Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-54681

Resharding recipient shards which are also donor may fail retryable writes with IncompleteTransactionHistory too early

    XMLWordPrintable

Details

    • Fully Compatible
    • ALL
    • Hide

      python buildscripts/resmoke.py run --suite=sharding resharding_inplace_retryable_writes.js
      

      resharding_inplace_retryable_writes.js

      (function() {
      "use strict";
       
      load("jstests/sharding/libs/resharding_test_fixture.js");
       
      const reshardingTest = new ReshardingTest({numDonors: 2, numRecipients: 2, reshardInPlace: true});
      reshardingTest.setup();
       
      const donorShardNames = reshardingTest.donorShardNames;
      const sourceCollection = reshardingTest.createShardedCollection({
          ns: "reshardingDb.coll",
          shardKeyPattern: {oldKey: 1},
          chunks: [
              {min: {oldKey: MinKey}, max: {oldKey: 0}, shard: donorShardNames[0]},
              {min: {oldKey: 0}, max: {oldKey: MaxKey}, shard: donorShardNames[1]},
          ],
      });
       
      assert.commandWorked(sourceCollection.insert([
          {_id: "stays on shard0", oldKey: -10, newKey: -10, counter: 0},
          {_id: "moves to shard0", oldKey: 10, newKey: -10, counter: 0},
      ]));
       
      const mongos = sourceCollection.getMongo();
      const session = mongos.startSession({causalConsistency: false, retryWrites: false});
      const sessionCollection = session.getDatabase(sourceCollection.getDB().getName())
                                    .getCollection(sourceCollection.getName());
       
      function runRetryableWrite(phase, expectedErrorCode = ErrorCodes.OK) {
          const res = sessionCollection.runCommand("update", {
              updates: [
                  {q: {_id: "stays on shard0"}, u: {$inc: {counter: 1}}},
                  {q: {_id: "moves to shard0"}, u: {$inc: {counter: 1}}},
              ],
              txnNumber: NumberLong(1)
          });
       
          if (expectedErrorCode === ErrorCodes.OK) {
              assert.commandWorked(res);
          } else {
              assert.commandFailedWithCode(res, expectedErrorCode);
          }
       
          const docs = sourceCollection.find().toArray();
          assert.eq(2, docs.length, {docs});
       
          for (const doc of docs) {
              assert.eq(1,
                        doc.counter,
                        {message: `retryable write executed more than once ${phase}`, id: doc._id, docs});
          }
      }
       
      runRetryableWrite("before resharding");
       
      const recipientShardNames = reshardingTest.recipientShardNames;
      reshardingTest.withReshardingInBackground(  //
          {
              newShardKeyPattern: {newKey: 1},
              newChunks: [
                  {min: {newKey: MinKey}, max: {newKey: 0}, shard: recipientShardNames[0]},
                  {min: {newKey: 0}, max: {newKey: MaxKey}, shard: recipientShardNames[1]},
              ],
          },
          () => {
              runRetryableWrite("during resharding");
       
              assert.soon(() => {
                  const coordinatorDoc = mongos.getCollection("config.reshardingOperations").findOne({
                      nss: sourceCollection.getFullName()
                  });
       
                  return coordinatorDoc !== null && coordinatorDoc.fetchTimestamp !== undefined;
              });
       
              runRetryableWrite("during resharding after fetchTimestamp was chosen");
       
              assert.soon(() => {
                  const coordinatorDoc = mongos.getCollection("config.reshardingOperations").findOne({
                      nss: sourceCollection.getFullName()
                  });
       
                  return coordinatorDoc !== null && coordinatorDoc.state === "applying";
              });
       
              runRetryableWrite("during resharding after collection cloning had finished");
          });
       
      runRetryableWrite("after resharding", ErrorCodes.IncompleteTransactionHistory);
       
      reshardingTest.teardown();
      })();
      

      Show
      python buildscripts/resmoke.py run --suite=sharding resharding_inplace_retryable_writes.js resharding_inplace_retryable_writes.js ( function () { "use strict" ;   load( "jstests/sharding/libs/resharding_test_fixture.js" );   const reshardingTest = new ReshardingTest({numDonors: 2, numRecipients: 2, reshardInPlace: true }); reshardingTest.setup();   const donorShardNames = reshardingTest.donorShardNames; const sourceCollection = reshardingTest.createShardedCollection({ ns: "reshardingDb.coll" , shardKeyPattern: {oldKey: 1}, chunks: [ {min: {oldKey: MinKey}, max: {oldKey: 0}, shard: donorShardNames[0]}, {min: {oldKey: 0}, max: {oldKey: MaxKey}, shard: donorShardNames[1]}, ], });   assert.commandWorked(sourceCollection.insert([ {_id: "stays on shard0" , oldKey: -10, newKey: -10, counter: 0}, {_id: "moves to shard0" , oldKey: 10, newKey: -10, counter: 0}, ]));   const mongos = sourceCollection.getMongo(); const session = mongos.startSession({causalConsistency: false , retryWrites: false }); const sessionCollection = session.getDatabase(sourceCollection.getDB().getName()) .getCollection(sourceCollection.getName());   function runRetryableWrite(phase, expectedErrorCode = ErrorCodes.OK) { const res = sessionCollection.runCommand( "update" , { updates: [ {q: {_id: "stays on shard0" }, u: {$inc: {counter: 1}}}, {q: {_id: "moves to shard0" }, u: {$inc: {counter: 1}}}, ], txnNumber: NumberLong(1) });   if (expectedErrorCode === ErrorCodes.OK) { assert.commandWorked(res); } else { assert.commandFailedWithCode(res, expectedErrorCode); }   const docs = sourceCollection.find().toArray(); assert.eq(2, docs.length, {docs});   for (const doc of docs) { assert.eq(1, doc.counter, {message: `retryable write executed more than once ${phase}`, id: doc._id, docs}); } }   runRetryableWrite( "before resharding" );   const recipientShardNames = reshardingTest.recipientShardNames; reshardingTest.withReshardingInBackground( // { newShardKeyPattern: {newKey: 1}, newChunks: [ {min: {newKey: MinKey}, max: {newKey: 0}, shard: recipientShardNames[0]}, {min: {newKey: 0}, max: {newKey: MaxKey}, shard: recipientShardNames[1]}, ], }, () => { runRetryableWrite( "during resharding" );   assert.soon(() => { const coordinatorDoc = mongos.getCollection( "config.reshardingOperations" ).findOne({ nss: sourceCollection.getFullName() });   return coordinatorDoc !== null && coordinatorDoc.fetchTimestamp !== undefined; });   runRetryableWrite( "during resharding after fetchTimestamp was chosen" );   assert.soon(() => { const coordinatorDoc = mongos.getCollection( "config.reshardingOperations" ).findOne({ nss: sourceCollection.getFullName() });   return coordinatorDoc !== null && coordinatorDoc.state === "applying" ; });   runRetryableWrite( "during resharding after collection cloning had finished" ); });   runRetryableWrite( "after resharding" , ErrorCodes.IncompleteTransactionHistory);   reshardingTest.teardown(); })();
    • Sharding 2021-03-08, Sharding 2021-03-22
    • 1

    Description

      The RecipientStateMachine skips constructing a ReshardingTxnCloner for itself when the recipient shard is also a donor shard. The intention was to allow retryable writes started before the resharding operation to remain retryable during the resharding operation. However, this approach is insufficient because a recipient shard may write the kIncompleteHistoryStmtId entry due to the config.transactions entry of another donor shard that had also participated in the same retryable write.

      void ReshardingRecipientService::RecipientStateMachine::_initTxnCloner(
          OperationContext* opCtx, const Timestamp& fetchTimestamp) {
          auto catalogCache = Grid::get(opCtx)->catalogCache();
          auto routingInfo = catalogCache->getShardedCollectionRoutingInfo(opCtx, _recipientDoc.getNss());
          std::set<ShardId> shardList;
       
          const auto myShardId = ShardingState::get(opCtx)->shardId();
          routingInfo.getAllShardIds(&shardList);
          shardList.erase(myShardId);
       
          for (const auto& shard : shardList) {
              _txnCloners.push_back(
                  std::make_unique<ReshardingTxnCloner>(ReshardingSourceId(_id, shard), fetchTimestamp));
          }
      }
      

      https://github.com/mongodb/mongo/blob/2b397f74e6a05b7c11e63a61d4aaf9443f58b150/src/mongo/db/s/resharding/resharding_recipient_service.cpp#L369

      Attachments

        Issue Links

          Activity

            People

              alex.taskov@mongodb.com Alexander Taskov (Inactive)
              max.hirschhorn@mongodb.com Max Hirschhorn
              Votes:
              0 Vote for this issue
              Watchers:
              3 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved: