-
Type:
Bug
-
Resolution: Unresolved
-
Priority:
Major - P3
-
None
-
Affects Version/s: 3.6.4
-
Component/s: Operations Layer
Problem
Under high load on database, connection errors when primary changes.
Some examples with three nodes replica set:
1) Switch off the primary node of three nodes.
2) with one node down, and two up, a primary and a secondary, switch on again the node. When primary changes, the connection close and the program crashes.
Configuration to reproduce:
mongodb driver version 3.6.4
node 12
Replica set defined by this connection string:
mongodb://mongo1,mongo2,mongo3/lora?replicaSet=rs0
Replica set configured with docker for the three containers mongo1, mongo2 and mongo3 using same image
image: mongo: 4.0
Replica set configuration is :
var cfg = { "_id": "rs0", "version": 1, "members": [ { "_id": 0, "host": "mongo1:27017", "priority": 4 }, { "_id": 1, "host": "mongo2:27017", "priority": 2 }, { "_id": 2, "host": "mongo3:27017", "priority": 1 } ] };
I made a simple script to reproduce the problem:
const MongoClient = require('mongodb').MongoClient; const assert = require('assert');// Connection URL var url = 'mongodb://mongo1,mongo2,mongo3/lora?replicaSet=rs0';// Database Name const dbName = 'test-enot';const options = { useUnifiedTopology: true, retryWrites: true, useNewUrlParser: true }const intervalTime = 1000;// Use connect method to connect to the server MongoClient.connect(url, options, function(err, client) { assert.strictEqual(null, err); console.log("Connected successfully to server"); const db = client.db(dbName); setInterval(() => { insertDocuments(db, function() { updateDocument(db, function() { removeDocument(db, function() { }); }); }); }, intervalTime); }); const insertDocuments = function(db, callback) { // Get the documents collection const collection = db.collection('documents'); // Insert some documents collection.insertMany([ {a : 1}, {a : 2}, {a : 3} ], function(err, result) { assert.strictEqual(err, null); assert.strictEqual(3, result.result.n); assert.strictEqual(3, result.ops.length); console.log("Inserted 3 documents into the collection"); callback(result); }); } const updateDocument = function(db, callback) { // Get the documents collection const collection = db.collection('documents'); // Update document where a is 2, set b equal to 1 collection.updateOne({ a : 2 } , { $set: { b : 1 } }, function(err, result) { assert.strictEqual(err, null); assert.strictEqual(1, result.result.n); console.log("Updated the document with the field a equal to 2"); callback(result); }); } const removeDocument = function(db, callback) { // Get the documents collection const collection = db.collection('documents'); // Delete document where a is 3 collection.deleteOne({ a : 3 }, function(err, result) { assert.strictEqual(err, null); assert.strictEqual(1, result.result.n); console.log("Removed the document with the field a equal to 3"); callback(result); }); }
With an intervalTime of 10 seconds (10000) the problem is not happening. The topology changes, election is made, new primary is chosen and primary changes without any problem. No errors are given.
With an interval time of 1 second (1000) the problem happens quite often, not always but almost.
I got this error:
actual: MongoError: not master
Then I added the dbOptions values described in the script ( useUnifiedTopology: true, retryWrites: true, useNewUrlParser: true) and the error message is different but happens in the same moment:
mongo-enot_1 | /usr/src/app/node_modules/mongodb/lib/utils.js:106 mongo-enot_1 | throw err; mongo-enot_1 | ^ mongo-enot_1 | mongo-enot_1 | AssertionError [ERR_ASSERTION]: Expected values to be strictly equal: mongo-enot_1 | + actual - expected mongo-enot_1 | mongo-enot_1 | + MongoNetworkError: connection 2 to 172.28.0.13:27017 closed mongo-enot_1 | + at /usr/src/app/node_modules/mongodb/lib/cmap/connection.js:68:15 mongo-enot_1 | + at Map.forEach (<anonymous>) mongo-enot_1 | + at Socket.<anonymous> (/usr/src/app/node_modules/mongodb/lib/cmap/connection.js:67:20) mongo-enot_1 | + at Socket.emit (events.js:314:20) mongo-enot_1 | + at TCP.<anonymous> (net.js:675:12) mongo-enot_1 | - null mongo-enot_1 | at /usr/src/app/server.js:44:16 mongo-enot_1 | at executeCallback (/usr/src/app/node_modules/mongodb/lib/operations/execute_operation.js:70:5) mongo-enot_1 | at /usr/src/app/node_modules/mongodb/lib/operations/insert_many.js:42:23 mongo-enot_1 | at /usr/src/app/node_modules/mongodb/lib/operations/bulk_write.js:70:16 mongo-enot_1 | at /usr/src/app/node_modules/mongodb/lib/utils.js:385:14 mongo-enot_1 | at executeCallback (/usr/src/app/node_modules/mongodb/lib/utils.js:375:25) mongo-enot_1 | at handleCallback (/usr/src/app/node_modules/mongodb/lib/utils.js:102:55) mongo-enot_1 | at resultHandler (/usr/src/app/node_modules/mongodb/lib/bulk/common.js:505:14) mongo-enot_1 | at handler (/usr/src/app/node_modules/mongodb/lib/core/sdam/topology.js:946:16) mongo-enot_1 | at /usr/src/app/node_modules/mongodb/lib/cmap/connection_pool.js:348:13 { mongo-enot_1 | generatedMessage: true, mongo-enot_1 | code: 'ERR_ASSERTION', mongo-enot_1 | actual: MongoNetworkError: connection 2 to 172.28.0.13:27017 closed mongo-enot_1 | at /usr/src/app/node_modules/mongodb/lib/cmap/connection.js:68:15 mongo-enot_1 | at Map.forEach (<anonymous>) mongo-enot_1 | at Socket.<anonymous> (/usr/src/app/node_modules/mongodb/lib/cmap/connection.js:67:20) mongo-enot_1 | at Socket.emit (events.js:314:20) mongo-enot_1 | at TCP.<anonymous> (net.js:675:12), mongo-enot_1 | expected: null, mongo-enot_1 | operator: 'strictEqual' mongo-enot_1 | }
¿Why is this happening?
¿Should implement any mechanism to prevent this?
¿Should not be the node driver agnostic to the connected node and just deal with the replica set connection?
