Retryable update may execute more than once if chunk is migrated and shard key pattern uses nested fields

XMLWordPrintableJSON

    • Fully Compatible
    • ALL
    • v5.1, v5.0, v4.4, v4.2
    • Hide 
      python buildscripts/resmoke.py run --suite=sharding repro_retryable_update_multiple_execution_dotted_sk.js
      repro_retryable_update_multiple_execution_dotted_sk.js
      (function() {
"use strict";

load("jstests/sharding/libs/create_sharded_collection_util.js");

const st = new ShardingTest({mongos: 1, config: 1, shards: 2, rs: {nodes: 1}});

const db = st.s.getDB("test");
const collection = db.getCollection("mycoll");
CreateShardedCollectionUtil.shardCollectionWithChunks(collection, {"x.y": 1}, [
    {min: {"x.y": MinKey}, max: {"x.y": 0}, shard: st.shard0.shardName},
    {min: {"x.y": 0}, max: {"x.y": 10}, shard: st.shard0.shardName},
    {min: {"x.y": 10}, max: {"x.y": 20}, shard: st.shard1.shardName},
    {min: {"x.y": 20}, max: {"x.y": MaxKey}, shard: st.shard1.shardName},
]);

assert.commandWorked(collection.insert({_id: 0, x: {y: 5}, counter: 0}));

const session = st.s.startSession({causalConsistency: false, retryWrites: false});
const sessionCollection = session.getDatabase(db.getName()).getCollection(collection.getName());

const updateCmd = {
    updates: [{q: {"x.y": 5, _id: 0}, u: {$inc: {counter: 1}}}],
    txnNumber: NumberLong(0),
};

const firstRes = assert.commandWorked(sessionCollection.runCommand("update", updateCmd));
assert.eq({n: firstRes.n, nModified: firstRes.nModified}, {n: 1, nModified: 1});

assert.commandWorked(db.adminCommand(
    {moveChunk: collection.getFullName(), find: {"x.y": 5}, to: st.shard1.shardName}));

const secondRes = assert.commandWorked(sessionCollection.runCommand("update", updateCmd));
print(`secondRes: ${tojsononeline(secondRes)}`);
assert.eq(collection.findOne({_id: 0}), {_id: 0, x: {y: 5}, counter: 1});

st.stop();
})();
      Show
      python buildscripts/resmoke.py run --suite=sharding repro_retryable_update_multiple_execution_dotted_sk.js repro_retryable_update_multiple_execution_dotted_sk.js ( function () { "use strict" ; load( "jstests/sharding/libs/create_sharded_collection_util.js" ); const st = new ShardingTest({mongos: 1, config: 1, shards: 2, rs: {nodes: 1}}); const db = st.s.getDB( "test" ); const collection = db.getCollection( "mycoll" ); CreateShardedCollectionUtil.shardCollectionWithChunks(collection, { "x.y" : 1}, [ {min: { "x.y" : MinKey}, max: { "x.y" : 0}, shard: st.shard0.shardName}, {min: { "x.y" : 0}, max: { "x.y" : 10}, shard: st.shard0.shardName}, {min: { "x.y" : 10}, max: { "x.y" : 20}, shard: st.shard1.shardName}, {min: { "x.y" : 20}, max: { "x.y" : MaxKey}, shard: st.shard1.shardName}, ]); assert.commandWorked(collection.insert({_id: 0, x: {y: 5}, counter: 0})); const session = st.s.startSession({causalConsistency: false , retryWrites: false }); const sessionCollection = session.getDatabase(db.getName()).getCollection(collection.getName()); const updateCmd = { updates: [{q: { "x.y" : 5, _id: 0}, u: {$inc: {counter: 1}}}], txnNumber: NumberLong(0), }; const firstRes = assert.commandWorked(sessionCollection.runCommand( "update" , updateCmd)); assert.eq({n: firstRes.n, nModified: firstRes.nModified}, {n: 1, nModified: 1}); assert.commandWorked(db.adminCommand( {moveChunk: collection.getFullName(), find: { "x.y" : 5}, to: st.shard1.shardName})); const secondRes = assert.commandWorked(sessionCollection.runCommand( "update" , updateCmd)); print( `secondRes: ${tojsononeline(secondRes)}` ); assert.eq(collection.findOne({_id: 0}), {_id: 0, x: {y: 5}, counter: 1}); st.stop(); })();
    • QE 2021-10-04, QE 2021-10-18
    • 165
    • None
    • 3
    • None
    • None
    • None
    • None
    • None
    • None

      Donor shards filter oplog entries relevant for session migration as part of chunk migration by extracting the shard key value from the oplog entry. In particular, it attempts to extract the shard key value from the 'o' field for op='i' insert oplog entries, from the 'o2' field for op='u' update oplog entries, and from the 'o' field for op='d' delete oplog entries. However, the shard key value has already been extracted from the document for op='u' and op='d' oplog entries as part of generating the oplog entries. For example, with a shard key pattern {"x.y": 1}, the resulting oplog entry would contain {o2: {"x.y": 5, _id: 0}} for an update and {o: {"x.y": 5, _id: 0}} for a delete. Attempting to extract the shard key value from those 'o2' and 'o' objects with ShardKeyPattern::extractShardKeyFromDoc() would result in a shard key value {"x.y": null} and lead the donor shard to incorrectly conclude the oplog entries isn't relevant for the chunk actively being migrated. This causes the recipient shard to not know the statement(s) from that oplog entry have already executed and therefore allows them to execute for a second time after the chunk migration commits.

      if (nextOplog->isCrudOpType()) {
    auto shardKey =
        _keyPattern.extractShardKeyFromDoc(nextOplog->getObjectContainingDocumentKey());
    if (!_chunkRange.containsKey(shardKey)) {
        continue;
    }
}

      [js_test:repro_retryable_update_multiple_execution_dotted_sk] uncaught exception: Error: [{ "_id" : 0, "x" : { "y" : 5 }, "counter" : 2 }] != [{ "_id" : 0, "x" : { "y" : 5 }, "counter" : 1 }] are not equal :

            Assignee:
            Bobby Morck (Inactive)
            Reporter:
            Max Hirschhorn
            Votes:
            0 Vote for this issue
            Watchers:
            11 Start watching this issue

              Created:
              Updated:
              Resolved: