$ cat /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 127.0.0.1 mongo1.example.com 127.0.0.1 mongo2.example.com 127.0.0.1 mongo3.example.com $ mongod --dbpath data/db1/ --replSet test1 --port 3333 --logpath /tmp/mongo1.log --logappend $ mongod --dbpath data/db2/ --replSet test1 --port 3334 --logpath /tmp/mongo2.log --logappend $ mongod --dbpath data/db3/ --replSet test1 --port 3335 --logpath /tmp/mongo3.log --logappend $ mongo --port 3333 --host mongo1.example.com rs.initiate( { _id : "test1", members: [{ _id : 0, host : "mongo1.example.com:3333" }, { _id : 1, host : "mongo2.example.com:3334" }, { _id : 2, host : "mongo3.example.com:3335" } ] } ) MongoDB Enterprise test1:SECONDARY> rs.status() { "set" : "test1", "date" : ISODate("2022-05-18T12:41:53.545Z"), "myState" : 1, "term" : NumberLong(1), "syncSourceHost" : "", "syncSourceId" : -1, "heartbeatIntervalMillis" : NumberLong(2000), "majorityVoteCount" : 2, "writeMajorityCount" : 2, "votingMembersCount" : 3, "writableVotingMembersCount" : 3, "optimes" : { "lastCommittedOpTime" : { "ts" : Timestamp(1652877709, 1), "t" : NumberLong(1) }, "lastCommittedWallTime" : ISODate("2022-05-18T12:41:49.865Z"), "readConcernMajorityOpTime" : { "ts" : Timestamp(1652877709, 1), "t" : NumberLong(1) }, "readConcernMajorityWallTime" : ISODate("2022-05-18T12:41:49.865Z"), "appliedOpTime" : { "ts" : Timestamp(1652877709, 1), "t" : NumberLong(1) }, "durableOpTime" : { "ts" : Timestamp(1652877709, 1), "t" : NumberLong(1) }, "lastAppliedWallTime" : ISODate("2022-05-18T12:41:49.865Z"), "lastDurableWallTime" : ISODate("2022-05-18T12:41:49.865Z") }, "lastStableRecoveryTimestamp" : Timestamp(1652877689, 4), "electionCandidateMetrics" : { "lastElectionReason" : "electionTimeout", "lastElectionDate" : ISODate("2022-05-18T12:41:29.813Z"), "electionTerm" : NumberLong(1), "lastCommittedOpTimeAtElection" : { "ts" : Timestamp(0, 0), "t" : NumberLong(-1) }, "lastSeenOpTimeAtElection" : { "ts" : Timestamp(1652877678, 1), "t" : NumberLong(-1) }, "numVotesNeeded" : 2, "priorityAtElection" : 1, "electionTimeoutMillis" : NumberLong(10000), "numCatchUpOps" : NumberLong(0), "newTermStartDate" : ISODate("2022-05-18T12:41:29.855Z"), "wMajorityWriteAvailabilityDate" : ISODate("2022-05-18T12:41:30.625Z") }, "members" : [ { "_id" : 0, "name" : "mongo1.example.com:3333", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 62, "optime" : { "ts" : Timestamp(1652877709, 1), "t" : NumberLong(1) }, "optimeDate" : ISODate("2022-05-18T12:41:49Z"), "lastAppliedWallTime" : ISODate("2022-05-18T12:41:49.865Z"), "lastDurableWallTime" : ISODate("2022-05-18T12:41:49.865Z"), "syncSourceHost" : "", "syncSourceId" : -1, "infoMessage" : "", "electionTime" : Timestamp(1652877689, 1), "electionDate" : ISODate("2022-05-18T12:41:29Z"), "configVersion" : 1, "configTerm" : 1, "self" : true, "lastHeartbeatMessage" : "" }, { "_id" : 1, "name" : "mongo2.example.com:3334", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 34, "optime" : { "ts" : Timestamp(1652877709, 1), "t" : NumberLong(1) }, "optimeDurable" : { "ts" : Timestamp(1652877709, 1), "t" : NumberLong(1) }, "optimeDate" : ISODate("2022-05-18T12:41:49Z"), "optimeDurableDate" : ISODate("2022-05-18T12:41:49Z"), "lastAppliedWallTime" : ISODate("2022-05-18T12:41:49.865Z"), "lastDurableWallTime" : ISODate("2022-05-18T12:41:49.865Z"), "lastHeartbeat" : ISODate("2022-05-18T12:41:51.829Z"), "lastHeartbeatRecv" : ISODate("2022-05-18T12:41:52.843Z"), "pingMs" : NumberLong(0), "lastHeartbeatMessage" : "", "syncSourceHost" : "mongo1.example.com:3333", "syncSourceId" : 0, "infoMessage" : "", "configVersion" : 1, "configTerm" : 1 }, { "_id" : 2, "name" : "mongo3.example.com:3335", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 34, "optime" : { "ts" : Timestamp(1652877709, 1), "t" : NumberLong(1) }, "optimeDurable" : { "ts" : Timestamp(1652877709, 1), "t" : NumberLong(1) }, "optimeDate" : ISODate("2022-05-18T12:41:49Z"), "optimeDurableDate" : ISODate("2022-05-18T12:41:49Z"), "lastAppliedWallTime" : ISODate("2022-05-18T12:41:49.865Z"), "lastDurableWallTime" : ISODate("2022-05-18T12:41:49.865Z"), "lastHeartbeat" : ISODate("2022-05-18T12:41:51.828Z"), "lastHeartbeatRecv" : ISODate("2022-05-18T12:41:52.836Z"), "pingMs" : NumberLong(0), "lastHeartbeatMessage" : "", "syncSourceHost" : "mongo1.example.com:3333", "syncSourceId" : 0, "infoMessage" : "", "configVersion" : 1, "configTerm" : 1 } ], "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1652877709, 1), "signature" : { "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="), "keyId" : NumberLong(0) } }, "operationTime" : Timestamp(1652877709, 1) } MongoDB Enterprise test1:PRIMARY> ############################ Shutdown all the replica set nodes############ ###########################Now delete the DNS records for all 3 nodes ##### [root@ip-172-31-34-77 ec2-user]# cat /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 #127.0.0.1 mongo1.example.com #127.0.0.1 mongo2.example.com #127.0.0.1 mongo3.example.com ############################ start all replica set nodes#################### $ grep -i isself /tmp/mongo1.log {"t":{"$date":"2022-05-18T12:43:26.010+00:00"},"s":"I", "c":"NETWORK", "id":4834700, "ctx":"ReplCoord-0","msg":"isSelf could not connect via connectSocketOnly","attr":{"hostAndPort":"mongo2.example.com:3334","error":{"code":6,"codeName":"HostUnreachable","errmsg":"couldn't connect to server mongo2.example.com:3334, connection attempt failed: HostNotFound: Could not find address for mongo2.example.com:3334: SocketException: Host not found (authoritative)"}}} {"t":{"$date":"2022-05-18T12:43:26.037+00:00"},"s":"I", "c":"NETWORK", "id":4834700, "ctx":"ReplCoord-0","msg":"isSelf could not connect via connectSocketOnly","attr":{"hostAndPort":"mongo3.example.com:3335","error":{"code":6,"codeName":"HostUnreachable","errmsg":"couldn't connect to server mongo3.example.com:3335, connection attempt failed: HostNotFound: Could not find address for mongo3.example.com:3335: SocketException: Host not found (authoritative)"}}} {"t":{"$date":"2022-05-18T12:46:51.677+00:00"},"s":"I", "c":"NETWORK", "id":4834700, "ctx":"ReplCoord-0","msg":"isSelf could not connect via connectSocketOnly","attr":{"hostAndPort":"mongo1.example.com:3333","error":{"code":6,"codeName":"HostUnreachable","errmsg":"couldn't connect to server mongo1.example.com:3333, connection attempt failed: HostNotFound: Could not find address for mongo1.example.com:3333: SocketException: Host not found (authoritative)"}}} {"t":{"$date":"2022-05-18T12:46:51.717+00:00"},"s":"I", "c":"NETWORK", "id":4834700, "ctx":"ReplCoord-0","msg":"isSelf could not connect via connectSocketOnly","attr":{"hostAndPort":"mongo2.example.com:3334","error":{"code":6,"codeName":"HostUnreachable","errmsg":"couldn't connect to server mongo2.example.com:3334, connection attempt failed: HostNotFound: Could not find address for mongo2.example.com:3334: SocketException: Host not found (authoritative)"}}} {"t":{"$date":"2022-05-18T12:46:51.722+00:00"},"s":"I", "c":"NETWORK", "id":4834700, "ctx":"ReplCoord-0","msg":"isSelf could not connect via connectSocketOnly","attr":{"hostAndPort":"mongo3.example.com:3335","error":{"code":6,"codeName":"HostUnreachable","errmsg":"couldn't connect to server mongo3.example.com:3335, connection attempt failed: HostNotFound: Could not find address for mongo3.example.com:3335: SocketException: Host not found (authoritative)"}}} $ tail -f /tmp/mongo1.log {"t":{"$date":"2022-05-18T12:46:51.722+00:00"},"s":"I", "c":"REPL", "id":21394, "ctx":"ReplCoord-0","msg":"This node is not a member of the config"} {"t":{"$date":"2022-05-18T12:46:51.722+00:00"},"s":"I", "c":"REPL", "id":21358, "ctx":"ReplCoord-0","msg":"Replica set state transition","attr":{"newState":"REMOVED","oldState":"STARTUP"}} {"t":{"$date":"2022-05-18T12:46:51.722+00:00"},"s":"I", "c":"REPL", "id":21320, "ctx":"ReplCoord-0","msg":"Updated term","attr":{"term":2}} {"t":{"$date":"2022-05-18T12:47:51.623+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"WTCheckpointThread","msg":"WiredTiger message","attr":{"message":"[1652878071:622956][1328605:0x7fe3b8eb7700], WT_SESSION.checkpoint: [WT_VERB_CHECKPOINT_PROGRESS] saving checkpoint snapshot min: 8, snapshot max: 8 snapshot count: 0, oldest timestamp: (1652877759, 1) , meta checkpoint timestamp: (1652877759, 1) base write gen: 32"}} {"t":{"$date":"2022-05-18T12:48:51.636+00:00"},"s":"I", "c":"STORAGE", "id":22430, "ctx":"WTCheckpointThread","msg":"WiredTiger message","attr":{"message":"[1652878131:636692][1328605:0x7fe3b8eb7700], WT_SESSION.checkpoint: [WT_VERB_CHECKPOINT_PROGRESS] saving checkpoint snapshot min: 10, snapshot max: 10 snapshot count: 0, oldest timestamp: (1652877759, 1) , meta checkpoint timestamp: (1652877759, 1) base write gen: 32"}} ###### Restore DNS records ############ [root@ip-172-31-34-77 ec2-user]# cat /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 127.0.0.1 mongo1.example.com 127.0.0.1 mongo2.example.com 127.0.0.1 mongo3.example.com ######## Now waited for exactly 5 minutes and connected to mongo1.example.com ######### ######## Only to find out the replica set remained in error state ######## $ mongo --host mongo1.example.com --port 3333 MongoDB shell version v4.4.14 connecting to: mongodb://mongo1.example.com:3333/?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("9a90577e-e13a-44e6-94f9-daf2b04645f0") } MongoDB server version: 4.4.14 --- The server generated these startup warnings when booting: 2022-05-18T12:46:51.621+00:00: Access control is not enabled for the database. Read and write access to data and configuration is unrestricted 2022-05-18T12:46:51.621+00:00: This server is bound to localhost. Remote systems will be unable to connect to this server. Start the server with --bind_ip
to specify which IP addresses it should serve responses from, or with --bind_ip_all to bind to all interfaces. If this behavior is desired, start the server with --bind_ip 127.0.0.1 to disable this warning 2022-05-18T12:46:51.621+00:00: /sys/kernel/mm/transparent_hugepage/enabled is 'always'. We suggest setting it to 'never' 2022-05-18T12:46:51.621+00:00: Soft rlimits too low 2022-05-18T12:46:51.621+00:00: currentValue: 1024 2022-05-18T12:46:51.621+00:00: recommendedMinimum: 64000 --- MongoDB Enterprise > rs.status() { "ok" : 0, "errmsg" : "Our replica set config is invalid or we are not a member of it", "code" : 93, "codeName" : "InvalidReplicaSetConfig" } MongoDB Enterprise >