commit 4abd14eb5963150d5b3cc178b0b02d930f13a5bd Author: Suganthi Mani Date: Mon Dec 16 14:57:52 2019 -0500 Fail to update rollback id - repro diff --git a/jstests/replsets/rollback_repro.js b/jstests/replsets/rollback_repro.js new file mode 100644 index 0000000000..719e096877 --- /dev/null +++ b/jstests/replsets/rollback_repro.js @@ -0,0 +1,66 @@ +/** + * This test demonstrates that a rollback immediately after disabling majority reads succeeds. + * @tags: [requires_persistence] + */ +(function() { +"use strict"; +load("jstests/replsets/libs/rollback_test.js"); +load("jstests/replsets/rslib.js"); + +const name = "rollback_test"; +const dbName = "test"; +const collName = "coll"; +TestData.allowUncleanShutdowns = true; +TestData.rollbackShutdowns = true; + +jsTest.log("Set up a Rollback Test with enableMajorityReadConcern=true"); +const replTest = new ReplSetTest( + {name, nodes: 3, useBridge: true, nodeOptions: {enableMajorityReadConcern: "true"}}); +replTest.startSet(); +let config = replTest.getReplSetConfig(); +config.members[2].priority = 0; +config.settings = { + chainingAllowed: false +}; +replTest.initiate(config); +const rollbackTest = new RollbackTest(name, replTest); + +const rollbackNode = rollbackTest.transitionToRollbackOperations(); +assert.commandWorked( + rollbackNode.getDB(dbName).runCommand({insert: collName, documents: [{_id: "rollback op"}]})); + +jsTest.log("Restart the rollback node with enableMajorityReadConcern=false"); +rollbackTest.restartNode(0, 15, {enableMajorityReadConcern: "false"}); + +rollbackTest.transitionToSyncSourceOperationsBeforeRollback(); + +jsTest.log(" Make rollback to hang after oplog truncation"); +assert.commandWorked(rollbackNode.adminCommand( + {configureFailPoint: "rollbackExitEarlyAfterOplogTruncate", mode: "alwaysOn"})); +const lastRBID = assert.commandWorked(rollbackNode.adminCommand("replSetGetRBID")).rbid; + +rollbackTest.transitionToSyncSourceOperationsDuringRollback(); + +jsTest.log("wait for the message"); +checkLog.contains(rollbackNode, "rollbackExitEarlyAfterOplogTruncate fail point enabled"); + +jsTest.log("Restart the node again"); +rollbackTest.restartNode(0, 9, {enableMajorityReadConcern: "false"}); + +jsTest.log("wait for Rollback id to increment"); +let rbid = -1; +assert.soon(() => { + try { + rbid = assert.commandWorked(rollbackNode.adminCommand("replSetGetRBID")).rbid; + } catch (e) { + // Command can fail when sync source is being cleared. + } + // Fail early if the rbid is greater than lastRBID+1. + assert.lte( + rbid, lastRBID + 1, `RBID is too large. current RBID: ${rbid}, last RBID: ${lastRBID}`); + + return rbid === lastRBID + 1; +}, "Timed out waiting for RBID to increment on " + rollbackNode.host, 60 * 1000); + +rollbackTest.stop(); +}()); diff --git a/src/mongo/db/repl/rs_rollback.cpp b/src/mongo/db/repl/rs_rollback.cpp index 5e3b43c70e..68b9d296be 100644 --- a/src/mongo/db/repl/rs_rollback.cpp +++ b/src/mongo/db/repl/rs_rollback.cpp @@ -95,6 +95,7 @@ using std::unique_ptr; namespace repl { MONGO_FAIL_POINT_DEFINE(rollbackExitEarlyAfterCollectionDrop); +MONGO_FAIL_POINT_DEFINE(rollbackExitEarlyAfterOplogTruncate); using namespace rollback_internal; @@ -1205,6 +1206,14 @@ Status _syncRollback(OperationContext* opCtx, }); syncFixUp(opCtx, how, rollbackSource, replCoord, replicationProcess); + if (MONGO_unlikely(rollbackExitEarlyAfterOplogTruncate.shouldFail())) { + log() << "rollbackExitEarlyAfterOplogTruncate fail point enabled. Returning early " + "until fail point is disabled."; + while (MONGO_unlikely(rollbackExitEarlyAfterOplogTruncate.shouldFail())) { + mongo::sleepsecs(1); + } + } + if (MONGO_unlikely(rollbackExitEarlyAfterCollectionDrop.shouldFail())) { log() << "rollbackExitEarlyAfterCollectionDrop fail point enabled. Returning early " "until fail point is disabled.";