Uploaded image for project: 'Node.js Driver'
  1. Node.js Driver
  2. NODE-3112

replica set resilience on election time after primary is down

    XMLWordPrintable

    Details

    • Type: Bug
    • Status: Open
    • Priority: Major - P3
    • Resolution: Unresolved
    • Affects Version/s: 3.6.4
    • Fix Version/s: None
    • Component/s: cmap
    • Labels:
      None

      Description

      Problem

      Under high load on database, connection errors when primary changes.

      Some examples with three nodes replica set:

      1) Switch off the primary node of three nodes.

      2) with one node down, and two up, a primary and a secondary, switch on again the node. When primary changes, the connection close and the program crashes.

       

      Configuration to reproduce:

      mongodb driver version 3.6.4

      node 12

       

      Replica set defined by this connection string:

      mongodb://mongo1,mongo2,mongo3/lora?replicaSet=rs0

       

       

       

      Replica set configured with docker for the three containers mongo1, mongo2 and mongo3 using same image

      image: mongo: 4.0

       

      Replica set configuration is :

        var cfg = {
          "_id": "rs0",
          "version": 1,
          "members": [
            {
              "_id": 0,
              "host": "mongo1:27017",
              "priority": 4
            },
            {
              "_id": 1,
              "host": "mongo2:27017",
              "priority": 2
            },
            {
              "_id": 2,
              "host": "mongo3:27017",
              "priority": 1
            }
          ]
        };
      

       

       

      I made a simple script to reproduce the problem:

       

      const MongoClient = require('mongodb').MongoClient;
      const assert = require('assert');// Connection URL
      var url = 'mongodb://mongo1,mongo2,mongo3/lora?replicaSet=rs0';// Database Name
      const dbName = 'test-enot';const options = {
          useUnifiedTopology: true,
          retryWrites: true,
          useNewUrlParser: true
      }const intervalTime = 1000;// Use connect method to connect to the server
      MongoClient.connect(url, options, function(err, client) {
          assert.strictEqual(null, err);
          console.log("Connected successfully to server");    const db = client.db(dbName);    setInterval(() => {
              insertDocuments(db, function() {
                  updateDocument(db, function() {
                      removeDocument(db, function() {
                      });
                  });
              });
          }, intervalTime);
      });
      const insertDocuments = function(db, callback) {
          // Get the documents collection
          const collection = db.collection('documents');
          // Insert some documents
          collection.insertMany([
              {a : 1}, {a : 2}, {a : 3}
          ], function(err, result) {
              assert.strictEqual(err, null);
              assert.strictEqual(3, result.result.n);
              assert.strictEqual(3, result.ops.length);
              console.log("Inserted 3 documents into the collection");
              callback(result);
          });
      }
      const updateDocument = function(db, callback) {
          // Get the documents collection
          const collection = db.collection('documents');
          // Update document where a is 2, set b equal to 1
          collection.updateOne({ a : 2 }
              , { $set: { b : 1 } }, function(err, result) {
              assert.strictEqual(err, null);
              assert.strictEqual(1, result.result.n);
              console.log("Updated the document with the field a equal to 2");
              callback(result);
          });
      }
      const removeDocument = function(db, callback) {
          // Get the documents collection
          const collection = db.collection('documents');
          // Delete document where a is 3
          collection.deleteOne({ a : 3 }, function(err, result) {
              assert.strictEqual(err, null);
              assert.strictEqual(1, result.result.n);
              console.log("Removed the document with the field a equal to 3");
              callback(result);
          });
      }
      

       

      With an intervalTime of 10 seconds (10000) the problem is not happening. The topology changes, election is made, new primary is chosen and primary changes without any problem. No errors are given.

       

      With an interval time of 1 second (1000) the problem happens quite often, not always but almost.

      I got this error:

      actual: MongoError: not master

      Then I added the dbOptions values described in the script ( useUnifiedTopology: true, retryWrites: true, useNewUrlParser: true) and the error message is different  but happens in the same moment:

      mongo-enot_1  | /usr/src/app/node_modules/mongodb/lib/utils.js:106
      mongo-enot_1  |       throw err;
      mongo-enot_1  |       ^
      mongo-enot_1  | 
      mongo-enot_1  | AssertionError [ERR_ASSERTION]: Expected values to be strictly equal:
      mongo-enot_1  | + actual - expected
      mongo-enot_1  | 
      mongo-enot_1  | + MongoNetworkError: connection 2 to 172.28.0.13:27017 closed
      mongo-enot_1  | +     at /usr/src/app/node_modules/mongodb/lib/cmap/connection.js:68:15
      mongo-enot_1  | +     at Map.forEach (<anonymous>)
      mongo-enot_1  | +     at Socket.<anonymous> (/usr/src/app/node_modules/mongodb/lib/cmap/connection.js:67:20)
      mongo-enot_1  | +     at Socket.emit (events.js:314:20)
      mongo-enot_1  | +     at TCP.<anonymous> (net.js:675:12)
      mongo-enot_1  | - null
      mongo-enot_1  |     at /usr/src/app/server.js:44:16
      mongo-enot_1  |     at executeCallback (/usr/src/app/node_modules/mongodb/lib/operations/execute_operation.js:70:5)
      mongo-enot_1  |     at /usr/src/app/node_modules/mongodb/lib/operations/insert_many.js:42:23
      mongo-enot_1  |     at /usr/src/app/node_modules/mongodb/lib/operations/bulk_write.js:70:16
      mongo-enot_1  |     at /usr/src/app/node_modules/mongodb/lib/utils.js:385:14
      mongo-enot_1  |     at executeCallback (/usr/src/app/node_modules/mongodb/lib/utils.js:375:25)
      mongo-enot_1  |     at handleCallback (/usr/src/app/node_modules/mongodb/lib/utils.js:102:55)
      mongo-enot_1  |     at resultHandler (/usr/src/app/node_modules/mongodb/lib/bulk/common.js:505:14)
      mongo-enot_1  |     at handler (/usr/src/app/node_modules/mongodb/lib/core/sdam/topology.js:946:16)
      mongo-enot_1  |     at /usr/src/app/node_modules/mongodb/lib/cmap/connection_pool.js:348:13 {
      mongo-enot_1  |   generatedMessage: true,
      mongo-enot_1  |   code: 'ERR_ASSERTION',
      mongo-enot_1  |   actual: MongoNetworkError: connection 2 to 172.28.0.13:27017 closed
      mongo-enot_1  |       at /usr/src/app/node_modules/mongodb/lib/cmap/connection.js:68:15
      mongo-enot_1  |       at Map.forEach (<anonymous>)
      mongo-enot_1  |       at Socket.<anonymous> (/usr/src/app/node_modules/mongodb/lib/cmap/connection.js:67:20)
      mongo-enot_1  |       at Socket.emit (events.js:314:20)
      mongo-enot_1  |       at TCP.<anonymous> (net.js:675:12),
      mongo-enot_1  |   expected: null,
      mongo-enot_1  |   operator: 'strictEqual'
      mongo-enot_1  | }
      

       

      ¿Why is this happening?
      ¿Should  implement any mechanism to prevent this?
      ¿Should not be the node driver agnostic to the connected node and just deal with the replica set connection?

       

       

       

        Attachments

          Issue Links

            Activity

              People

              Assignee:
              alexander.golin Alexander Golin
              Reporter:
              rji@loriot.io Raúl Jiménez
              Votes:
              0 Vote for this issue
              Watchers:
              4 Start watching this issue

                Dates

                Created:
                Updated: