Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-54681

Resharding recipient shards which are also donor may fail retryable writes with IncompleteTransactionHistory too early

XMLWordPrintableJSON

    • Fully Compatible
    • ALL
    • Hide 
      python buildscripts/resmoke.py run --suite=sharding resharding_inplace_retryable_writes.js
      resharding_inplace_retryable_writes.js
      (function() {
"use strict";

load("jstests/sharding/libs/resharding_test_fixture.js");

const reshardingTest = new ReshardingTest({numDonors: 2, numRecipients: 2, reshardInPlace: true});
reshardingTest.setup();

const donorShardNames = reshardingTest.donorShardNames;
const sourceCollection = reshardingTest.createShardedCollection({
    ns: "reshardingDb.coll",
    shardKeyPattern: {oldKey: 1},
    chunks: [
        {min: {oldKey: MinKey}, max: {oldKey: 0}, shard: donorShardNames[0]},
        {min: {oldKey: 0}, max: {oldKey: MaxKey}, shard: donorShardNames[1]},
    ],
});

assert.commandWorked(sourceCollection.insert([
    {_id: "stays on shard0", oldKey: -10, newKey: -10, counter: 0},
    {_id: "moves to shard0", oldKey: 10, newKey: -10, counter: 0},
]));

const mongos = sourceCollection.getMongo();
const session = mongos.startSession({causalConsistency: false, retryWrites: false});
const sessionCollection = session.getDatabase(sourceCollection.getDB().getName())
                              .getCollection(sourceCollection.getName());

function runRetryableWrite(phase, expectedErrorCode = ErrorCodes.OK) {
    const res = sessionCollection.runCommand("update", {
        updates: [
            {q: {_id: "stays on shard0"}, u: {$inc: {counter: 1}}},
            {q: {_id: "moves to shard0"}, u: {$inc: {counter: 1}}},
        ],
        txnNumber: NumberLong(1)
    });

    if (expectedErrorCode === ErrorCodes.OK) {
        assert.commandWorked(res);
    } else {
        assert.commandFailedWithCode(res, expectedErrorCode);
    }

    const docs = sourceCollection.find().toArray();
    assert.eq(2, docs.length, {docs});

    for (const doc of docs) {
        assert.eq(1,
                  doc.counter,
                  {message: `retryable write executed more than once ${phase}`, id: doc._id, docs});
    }
}

runRetryableWrite("before resharding");

const recipientShardNames = reshardingTest.recipientShardNames;
reshardingTest.withReshardingInBackground(  //
    {
        newShardKeyPattern: {newKey: 1},
        newChunks: [
            {min: {newKey: MinKey}, max: {newKey: 0}, shard: recipientShardNames[0]},
            {min: {newKey: 0}, max: {newKey: MaxKey}, shard: recipientShardNames[1]},
        ],
    },
    () => {
        runRetryableWrite("during resharding");

        assert.soon(() => {
            const coordinatorDoc = mongos.getCollection("config.reshardingOperations").findOne({
                nss: sourceCollection.getFullName()
            });

            return coordinatorDoc !== null && coordinatorDoc.fetchTimestamp !== undefined;
        });

        runRetryableWrite("during resharding after fetchTimestamp was chosen");

        assert.soon(() => {
            const coordinatorDoc = mongos.getCollection("config.reshardingOperations").findOne({
                nss: sourceCollection.getFullName()
            });

            return coordinatorDoc !== null && coordinatorDoc.state === "applying";
        });

        runRetryableWrite("during resharding after collection cloning had finished");
    });

runRetryableWrite("after resharding", ErrorCodes.IncompleteTransactionHistory);

reshardingTest.teardown();
})();
      Show
      python buildscripts/resmoke.py run --suite=sharding resharding_inplace_retryable_writes.js resharding_inplace_retryable_writes.js ( function () { "use strict" ; load( "jstests/sharding/libs/resharding_test_fixture.js" ); const reshardingTest = new ReshardingTest({numDonors: 2, numRecipients: 2, reshardInPlace: true }); reshardingTest.setup(); const donorShardNames = reshardingTest.donorShardNames; const sourceCollection = reshardingTest.createShardedCollection({ ns: "reshardingDb.coll" , shardKeyPattern: {oldKey: 1}, chunks: [ {min: {oldKey: MinKey}, max: {oldKey: 0}, shard: donorShardNames[0]}, {min: {oldKey: 0}, max: {oldKey: MaxKey}, shard: donorShardNames[1]}, ], }); assert.commandWorked(sourceCollection.insert([ {_id: "stays on shard0" , oldKey: -10, newKey: -10, counter: 0}, {_id: "moves to shard0" , oldKey: 10, newKey: -10, counter: 0}, ])); const mongos = sourceCollection.getMongo(); const session = mongos.startSession({causalConsistency: false , retryWrites: false }); const sessionCollection = session.getDatabase(sourceCollection.getDB().getName()) .getCollection(sourceCollection.getName()); function runRetryableWrite(phase, expectedErrorCode = ErrorCodes.OK) { const res = sessionCollection.runCommand( "update" , { updates: [ {q: {_id: "stays on shard0" }, u: {$inc: {counter: 1}}}, {q: {_id: "moves to shard0" }, u: {$inc: {counter: 1}}}, ], txnNumber: NumberLong(1) }); if (expectedErrorCode === ErrorCodes.OK) { assert.commandWorked(res); } else { assert.commandFailedWithCode(res, expectedErrorCode); } const docs = sourceCollection.find().toArray(); assert.eq(2, docs.length, {docs}); for ( const doc of docs) { assert.eq(1, doc.counter, {message: `retryable write executed more than once ${phase}` , id: doc._id, docs}); } } runRetryableWrite( "before resharding" ); const recipientShardNames = reshardingTest.recipientShardNames; reshardingTest.withReshardingInBackground( // { newShardKeyPattern: {newKey: 1}, newChunks: [ {min: {newKey: MinKey}, max: {newKey: 0}, shard: recipientShardNames[0]}, {min: {newKey: 0}, max: {newKey: MaxKey}, shard: recipientShardNames[1]}, ], }, () => { runRetryableWrite( "during resharding" ); assert.soon(() => { const coordinatorDoc = mongos.getCollection( "config.reshardingOperations" ).findOne({ nss: sourceCollection.getFullName() }); return coordinatorDoc !== null && coordinatorDoc.fetchTimestamp !== undefined; }); runRetryableWrite( "during resharding after fetchTimestamp was chosen" ); assert.soon(() => { const coordinatorDoc = mongos.getCollection( "config.reshardingOperations" ).findOne({ nss: sourceCollection.getFullName() }); return coordinatorDoc !== null && coordinatorDoc.state === "applying" ; }); runRetryableWrite( "during resharding after collection cloning had finished" ); }); runRetryableWrite( "after resharding" , ErrorCodes.IncompleteTransactionHistory); reshardingTest.teardown(); })();
    • Sharding 2021-03-08, Sharding 2021-03-22
    • 1

      The RecipientStateMachine skips constructing a ReshardingTxnCloner for itself when the recipient shard is also a donor shard. The intention was to allow retryable writes started before the resharding operation to remain retryable during the resharding operation. However, this approach is insufficient because a recipient shard may write the kIncompleteHistoryStmtId entry due to the config.transactions entry of another donor shard that had also participated in the same retryable write.

      void ReshardingRecipientService::RecipientStateMachine::_initTxnCloner(
    OperationContext* opCtx, const Timestamp& fetchTimestamp) {
    auto catalogCache = Grid::get(opCtx)->catalogCache();
    auto routingInfo = catalogCache->getShardedCollectionRoutingInfo(opCtx, _recipientDoc.getNss());
    std::set<ShardId> shardList;

    const auto myShardId = ShardingState::get(opCtx)->shardId();
    routingInfo.getAllShardIds(&shardList);
    shardList.erase(myShardId);

    for (const auto& shard : shardList) {
        _txnCloners.push_back(
            std::make_unique<ReshardingTxnCloner>(ReshardingSourceId(_id, shard), fetchTimestamp));
    }
}

      https://github.com/mongodb/mongo/blob/2b397f74e6a05b7c11e63a61d4aaf9443f58b150/src/mongo/db/s/resharding/resharding_recipient_service.cpp#L369

            Assignee:
            alex.taskov@mongodb.com Alexander Taskov (Inactive)
            Reporter:
            max.hirschhorn@mongodb.com Max Hirschhorn
            Votes:
            0 Vote for this issue
            Watchers:
            3 Start watching this issue

              Created:
              Updated:
              Resolved: