-
Type:
Bug
-
Resolution: Fixed
-
Priority:
Major - P3
-
Affects Version/s: None
-
Component/s: Testing Infrastructure
-
None
-
Fully Compatible
-
ALL
-
v5.2, v5.0
-
Server Serverless 2022-01-10, Server Serverless 2022-01-24
-
15
The inject_tenant_prefix.js override uses a Mongo.prototype.runCommand() override to send a command over the reroutingMongo replica set connection. However, Mongo.prototype._markNodeAsFailed() isn't being overridden by this file, so when the command fails with a retryable error from the reroutingMongo replica set connection, _markNodeAsFailed() is still called on the original replica set connection. The RSM for the original replica set connection ignores the error and won't attempt to discover a new primary because the host isn't actually part of that replica set. This leads the JavaScript test to continue its retries unsuccessfully against the stale primary of the rerouting replica set connection and ultimately fail the test.
[js_test:index_check6] | 2021-12-17T13:08:08.579Z I NETWORK 4712102 [js] "Host failed in replica set","attr":{"replicaSet":"rs0","host":"localhost:20753","error":{"code":91,"codeName":"ShutdownInProgress","errmsg":"Replication is being shut down"},"action":{"dropConnections":true,"requestImmediateCheck":true,"outcome":{"host":"localhost:20753","success":false,"errorMessage":"ShutdownInProgress: Replication is being shut down"}}}
[js_test:index_check6] | 2021-12-17T13:08:08.579Z I CONNPOOL 22572 [js] "Dropping all pooled connections","attr":{"hostAndPort":"localhost:20753","error":"PooledConnectionsDropped: Pooled connections dropped"}
[js_test:index_check6] | 2021-12-17T13:08:08.579Z I - 4333227 [js] "RSM monitoring host in expedited mode until we detect a primary","attr":{"host":"localhost:20751","replicaSet":"rs0"}
[js_test:index_check6] | 2021-12-17T13:08:08.579Z I - 4333227 [js] "RSM monitoring host in expedited mode until we detect a primary","attr":{"host":"localhost:20752","replicaSet":"rs0"}
[js_test:index_check6] | 2021-12-17T13:08:08.579Z I - 4333227 [js] "RSM monitoring host in expedited mode until we detect a primary","attr":{"host":"localhost:20750","replicaSet":"rs0"}
[js_test:index_check6] | 2021-12-17T13:08:08.580Z I NETWORK 20219 [js] "Ignoring isMaster reply from server that is not in the topology","attr":{"serverAddress":"localhost:20753"}
...
[j3:rs1:n0] | 2021-12-17T13:08:08.575+00:00 I REPL 21339 [conn102] "Replication failed for write concern","attr":{"status":"ShutdownInProgress: Replication is being shut down","writeConcern":{"w":"majority","j":true,"wtimeout":300321,"provenance":"clientSupplied"},"opTime":{"ts":{"$timestamp":{"t":1639746488,"i":19}},"t":7},"opID":4283,"allDurable":{"$timestamp":{"t":1639746488,"i":19}},"progress":{"lastCommittedOpTime":{"ts":{"$timestamp":{"t":1639746488,"i":17}},"t":7},"currentCommittedSnapshotOpTime":{"ts":{"$timestamp":{"t":1639746488,"i":17}},"t":7},"replicationProgress":[{"host":"localhost:20753","optime":{"ts":{"$timestamp":{"t":1639746488,"i":19}},"t":7},"lastAppliedOpTime":{"ts":{"$timestamp":{"t":1639746488,"i":19}},"t":7},"heartbeatAppliedOpTime":{"ts":{"$timestamp":{"t":0,"i":0}},"t":-1},"heartbeatDurableOpTime":{"ts":{"$timestamp":{"t":0,"i":0}},"t":-1},"memberId":0},{"host":"localhost:20754","optime":{"ts":{"$timestamp":{"t":1639746488,"i":17}},"t":7},"lastAppliedOpTime":{"ts":{"$timestamp":{"t":1639746488,"i":19}},"t":7},"heartbeatAppliedOpTime":{"ts":{"$timestamp":{"t":1639746487,"i":7}},"t":7},"heartbeatDurableOpTime":{"ts":{"$timestamp":{"t":1639746487,"i":7}},"t":7},"memberId":1},{"host":"localhost:20755","optime":{"ts":{"$timestamp":{"t":1639746488,"i":17}},"t":7},"lastAppliedOpTime":{"ts":{"$timestamp":{"t":1639746488,"i":17}},"t":7},"heartbeatAppliedOpTime":{"ts":{"$timestamp":{"t":1639746487,"i":7}},"t":7},"heartbeatDurableOpTime":{"ts":{"$timestamp":{"t":1639746487,"i":7}},"t":7},"memberId":2}]}}