Loading...

Type: Bug
Resolution: Done
Priority: Major - P3
Fix Version/s: None
Affects Version/s: None
Component/s: None
Labels:
None

Operating System:
ALL
Steps To Reproduce:
Hide

Login to any MongoDB pod in a kubernetes environment:
kubectl exec it <pod-name> - namespace mongo

Create a replica set with a primary member.

add an unreachable node
rs.add("mongo-1-mongo:27017")
rs.add("mongo-2-mongo:27017")

{ { "_id" : 7, "name" : "mongo-1-mongo:27017", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "ts" : Timestamp(0, 0), "t" : NumberLong(-1) }, "optimeDurable" : { "ts" : Timestamp(0, 0), "t" : NumberLong(-1) }, "optimeDate" : ISODate("1970-01-01T00:00:00Z"), "optimeDurableDate" : ISODate("1970-01-01T00:00:00Z"), "lastAppliedWallTime" : ISODate("1970-01-01T00:00:00Z"), "lastDurableWallTime" : ISODate("1970-01-01T00:00:00Z"), "lastHeartbeat" : ISODate("2022-03-24T10:04:04.439Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : NumberLong(0), "lastHeartbeatMessage" : "Error connecting to mongo-1-mongo:27017 :: caused by :: Could not find address for mongo-1-mongo:27017: SocketException: Host not found (authoritative)", "syncSourceHost" : "", "syncSourceId" : -1, "infoMessage" : "", "configVersion" : -1, "configTerm" : -1 }, { "_id" : 8, "name" : "mongo-2-mongo:27017", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "ts" : Timestamp(0, 0), "t" : NumberLong(-1) }, "optimeDurable" : { "ts" : Timestamp(0, 0), "t" : NumberLong(-1) }, "optimeDate" : ISODate("1970-01-01T00:00:00Z"), "optimeDurableDate" : ISODate("1970-01-01T00:00:00Z"), "lastAppliedWallTime" : ISODate("1970-01-01T00:00:00Z"), "lastDurableWallTime" : ISODate("1970-01-01T00:00:00Z"), "lastHeartbeat" : ISODate("2022-03-24T10:04:04.433Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : NumberLong(0), "lastHeartbeatMessage" : "Error connecting to mongo-2-mongo:27017 :: caused by :: Could not find address for mongo-2-mongo:27017: SocketException: Host not found (authoritative)", "syncSourceHost" : "", "syncSourceId" : -1, "infoMessage" : "", "configVersion" : -1, "configTerm" : -1 } ],

The two nodes should be unreachable nodes and then do
rs.remove("mongo-1-mongo:27017")
rs.remove("mongo-2-mongo:27017")

rs0:PRIMARY> rs.remove("mongo-2-mongo:27017"){ "ok" : 0, "errmsg" : "Cannot provide newlyAdded field to member config during reconfig.", "code" : 93, "codeName" : "InvalidReplicaSetConfig", "$clusterTime" : { "clusterTime" : Timestamp(1648116296, 1), "signature" : { "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="), "keyId" : NumberLong(0) } }, "operationTime" : Timestamp(1648116296, 1)} rs0:PRIMARY> rs.remove("mongo-1-mongo:27017"){ "ok" : 0, "errmsg" : "Cannot provide newlyAdded field to member config during reconfig.", "code" : 93, "codeName" : "InvalidReplicaSetConfig", "$clusterTime" : { "clusterTime" : Timestamp(1648116306, 1), "signature" : { "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="), "keyId" : NumberLong(0) } }, "operationTime" : Timestamp(1648116306, 1)} rs0:PRIMARY>
Show
Login to any MongoDB pod in a kubernetes environment: kubectl exec it <pod-name> - namespace mongo Create a replica set with a primary member. add an unreachable node rs.add("mongo-1-mongo:27017") rs.add("mongo-2-mongo:27017") { { "_id" : 7, "name" : "mongo-1-mongo:27017" , "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)" , "uptime" : 0, "optime" : { "ts" : Timestamp(0, 0), "t" : NumberLong(-1) }, "optimeDurable" : { "ts" : Timestamp(0, 0), "t" : NumberLong(-1) }, "optimeDate" : ISODate( "1970-01-01T00:00:00Z" ), "optimeDurableDate" : ISODate( "1970-01-01T00:00:00Z" ), "lastAppliedWallTime" : ISODate( "1970-01-01T00:00:00Z" ), "lastDurableWallTime" : ISODate( "1970-01-01T00:00:00Z" ), "lastHeartbeat" : ISODate( "2022-03-24T10:04:04.439Z" ), "lastHeartbeatRecv" : ISODate( "1970-01-01T00:00:00Z" ), "pingMs" : NumberLong(0), "lastHeartbeatMessage" : "Error connecting to mongo-1-mongo:27017 :: caused by :: Could not find address for mongo-1-mongo:27017: SocketException: Host not found (authoritative)" , "syncSourceHost" : "", " syncSourceId " : -1, " infoMessage " : " ", " configVersion " : -1, " configTerm " : -1 }, { " _id " : 8, " name " : " mongo-2-mongo:27017 ", " health " : 0, " state " : 8, " stateStr " : " (not reachable/healthy) ", " uptime " : 0, " optime " : { " ts " : Timestamp(0, 0), " t " : NumberLong(-1) }, " optimeDurable " : { " ts " : Timestamp(0, 0), " t " : NumberLong(-1) }, " optimeDate " : ISODate(" 1970-01-01T00:00:00Z "), " optimeDurableDate " : ISODate(" 1970-01-01T00:00:00Z "), " lastAppliedWallTime " : ISODate(" 1970-01-01T00:00:00Z "), " lastDurableWallTime " : ISODate(" 1970-01-01T00:00:00Z "), " lastHeartbeat " : ISODate(" 2022-03-24T10:04:04.433Z "), " lastHeartbeatRecv " : ISODate(" 1970-01-01T00:00:00Z "), " pingMs " : NumberLong(0), " lastHeartbeatMessage " : " Error connecting to mongo-2-mongo:27017 :: caused by :: Could not find address for mongo-2-mongo:27017: SocketException: Host not found (authoritative) ", " syncSourceHost " : " ", " syncSourceId " : -1, " infoMessage " : " ", " configVersion " : -1, " configTerm" : -1 } ], The two nodes should be unreachable nodes and then do rs.remove("mongo-1-mongo:27017") rs.remove("mongo-2-mongo:27017") rs0:PRIMARY> rs.remove( "mongo-2-mongo:27017" ){ "ok" : 0, "errmsg" : "Cannot provide newlyAdded field to member config during reconfig." , "code" : 93, "codeName" : "InvalidReplicaSetConfig" , "$clusterTime" : { "clusterTime" : Timestamp(1648116296, 1), "signature" : { "hash" : BinData(0, "AAAAAAAAAAAAAAAAAAAAAAAAAAA=" ), "keyId" : NumberLong(0) } }, "operationTime" : Timestamp(1648116296, 1)} rs0:PRIMARY> rs.remove( "mongo-1-mongo:27017" ){ "ok" : 0, "errmsg" : "Cannot provide newlyAdded field to member config during reconfig." , "code" : 93, "codeName" : "InvalidReplicaSetConfig" , "$clusterTime" : { "clusterTime" : Timestamp(1648116306, 1), "signature" : { "hash" : BinData(0, "AAAAAAAAAAAAAAAAAAAAAAAAAAA=" ), "keyId" : NumberLong(0) } }, "operationTime" : Timestamp(1648116306, 1)} rs0:PRIMARY>

A replica set deployed in an kubernetes environment, and tried to add two nodes which are unreachable were and added, but rs.remove() does not remove it once when logged in inside pod using mongo.

conf = {
    _id: "replset",
    members: [
        {_id: 0, host: "localhost:27017"}
    ]
}
rs.initiate(conf)// Adding and removing one nonexistent secondary should work
assert.commandWorked(rs.add("localhost:28018"))
assert.commandWorked(rs.remove("localhost:28018"))// Adding and removing two nonexistent secondaries does not work
assert.commandWorked(rs.add("localhost:28018"))
assert.commandWorked(rs.add("localhost:29019"))
assert.commandWorked(rs.remove("localhost:28018"))
assert.commandWorked(rs.remove("localhost:29019"))

Note: This does not depend on the k8 environment and this issue doesn't seem to affect mongosh but only the mongo shell.

I tested this on MongoDB v5.0.7.

Details

Description

Attachments

Activity

People

Dates