diff --git a/jstests/sharding/recover_multiple_migrations_on_stepup.js b/jstests/sharding/recover_multiple_migrations_on_stepup.js new file mode 100644 index 00000000000..7fb69046b06 --- /dev/null +++ b/jstests/sharding/recover_multiple_migrations_on_stepup.js @@ -0,0 +1,72 @@ +/** + * Tests that after a migration fails to complete (thus leaving a migrationCoordinator document + * pending to be recovered) and still has not been recovered, if a new migration to a different + * collection also fails and needs recovery, after a stepdown the shard will be able to properly + * recover both on stepup. + */ +(function() { +"use strict"; + +load("jstests/libs/fail_point_util.js"); +load('./jstests/libs/chunk_manipulation_util.js'); + +// Disable checking for index consistency to ensure that the config server doesn't trigger a +// StaleShardVersion exception on the shards and cause them to refresh their sharding metadata. That +// would interfere with the precise migration recovery interleaving this test requires. +const nodeOptions = { + setParameter: {enableShardedIndexConsistencyCheck: false} +}; + +var st = new ShardingTest({shards: 2, other: {configOptions: nodeOptions}}); +let staticMongod = MongoRunner.runMongod({}); + +const dbName = "test"; +const collNameA = "foo"; +const collNameB = "bar"; +const nsA = dbName + "." + collNameA; +const nsB = dbName + "." + collNameB; +const collA = st.s.getDB(dbName)[collNameA]; +const collB = st.s.getDB(dbName)[collNameB]; + +assert.commandWorked(st.s.adminCommand({enableSharding: dbName})); +st.ensurePrimaryShard(dbName, st.shard0.shardName); +assert.commandWorked(st.s.adminCommand({shardCollection: nsA, key: {_id: 1}})); +assert.commandWorked(st.s.adminCommand({shardCollection: nsB, key: {_id: 1}})); + +// Start a first migration and make it fail before deleting the migrationCoordinators recovery +// document. The shard does not trigger a best-effort shard version recovery in this error path, so +// the migration will only be recovered on the next access to the collection. +let failDeleteMigrationCoordinatorDocumentLocallyFailpoint = + configureFailPoint(st.rs0.getPrimary(), "failDeleteMigrationCoordinatorDocumentLocally"); + +// Succeeds because although the shard failed the move chunk, it committed on the configsvr. So when +// the configsvr sees the command failure, it refreshes, sees it committed and returns success to +// the router. +assert.commandWorked(st.s.adminCommand({moveChunk: nsA, find: {_id: 0}, to: st.shard1.shardName})); +failDeleteMigrationCoordinatorDocumentLocallyFailpoint.off(); + +// Start a second migration on a different collection, wait until it persists it's recovery document +// and then step down the donor. +let moveChunkHangAtStep3Failpoint = configureFailPoint(st.rs0.getPrimary(), "moveChunkHangAtStep3"); + +var joinMoveChunk2 = moveChunkParallel( + staticMongod, st.s0.host, {_id: 0}, null, nsB, st.shard1.shardName, false /* expectSuccess */); + +moveChunkHangAtStep3Failpoint.wait(); + +// Now we have two migrations pending to be recovered +assert.eq(2, st.shard0.getDB('config')['migrationCoordinators'].find().itcount()); + +// Stepdown the donor shard +assert.commandWorked(st.rs0.getPrimary().adminCommand({replSetStepDown: 5, force: true})); + +moveChunkHangAtStep3Failpoint.off(); +joinMoveChunk2(); + +// Check that the donor shard has been able to recover the shard version for both collections. +assert.eq(0, collA.find().itcount()); +assert.eq(0, collB.find().itcount()); + +MongoRunner.stopMongod(staticMongod); +st.stop(); +})(); diff --git a/src/mongo/db/s/migration_util.cpp b/src/mongo/db/s/migration_util.cpp index a8dfbbbe235..c55dc45f076 100644 --- a/src/mongo/db/s/migration_util.cpp +++ b/src/mongo/db/s/migration_util.cpp @@ -100,6 +100,7 @@ MONGO_FAIL_POINT_DEFINE(hangInReadyRangeDeletionLocallyInterruptible); MONGO_FAIL_POINT_DEFINE(hangInReadyRangeDeletionLocallyThenSimulateErrorUninterruptible); MONGO_FAIL_POINT_DEFINE(hangInAdvanceTxnNumInterruptible); MONGO_FAIL_POINT_DEFINE(hangInAdvanceTxnNumThenSimulateErrorUninterruptible); +MONGO_FAIL_POINT_DEFINE(failDeleteMigrationCoordinatorDocumentLocally); const char kSourceShard[] = "source"; const char kDestinationShard[] = "destination"; @@ -831,6 +832,11 @@ void deleteMigrationCoordinatorDocumentLocally(OperationContext* opCtx, const UU // shardVersion, which will ensure that it will see at least the same shardVersion. VectorClockMutable::get(opCtx)->waitForDurableConfigTime().get(opCtx); + if (MONGO_unlikely(failDeleteMigrationCoordinatorDocumentLocally.shouldFail())) { + uasserted(ErrorCodes::InternalError, + "Failed due to failDeleteMigrationCoordinatorDocumentLocally failpoint"); + } + PersistentTaskStore store( NamespaceString::kMigrationCoordinatorsNamespace); store.remove(opCtx,