Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-46651

Concurrent update with upsert:true and dropCollection can error

    • Type: Icon: Bug Bug
    • Resolution: Unresolved
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: None
    • Component/s: Write Ops
    • Labels:
    • Query Execution
    • ALL
    • Hide
      jstests/concurrency/fsm_workloads/CRUD_and_commands.js
      diff --git a/jstests/concurrency/fsm_workloads/CRUD_and_commands.js b/jstests/concurrency/fsm_workloads/CRUD_and_commands.js
      new file mode 100644
      index 0000000000..6fc472dd93
      --- /dev/null
      +++ b/jstests/concurrency/fsm_workloads/CRUD_and_commands.js
      @@ -0,0 +1,223 @@
      +'use strict';
      +
      +/**
      + * Perform CRUD operations, some of which may implicitly create collections. Also perform index
      + * creations which may implicitly create collections.
      + */
      +
      +var $config = (function() {
      +    const data = {numIds: 10, numDocsToInsertPerThread: 5, valueToBeInserted: 1, batchSize: 50};
      +
      +    const states = {
      +        init: function init(db, collName) {
      +            this.session = db.getMongo().startSession({causalConsistency: true});
      +            this.sessionDb = this.session.getDatabase(db.getName());
      +            this.docValue = "mydoc";
      +        },
      +
      +        insertDocs: function insertDocs(db, collName) {
      +            for (let i = 0; i < 5; i++) {
      +                const res = db[collName].insert({value: this.docValue, num: 1});
      +                assertWhenOwnColl.commandWorked(res);
      +                assertWhenOwnColl.eq(1, res.nInserted);
      +            }
      +        },
      +
      +        updateDocs: function updateDocs(db, collName) {
      +            for (let i = 0; i < 5; ++i) {
      +                let indexToUpdate = Math.floor(Math.random() * this.numIds);
      +                try {
      +                    assertWhenOwnColl.commandWorked(db[collName].update(
      +                        {_id: indexToUpdate}, {$inc: {num: 1}}, {upsert: true}));
      +                } catch (e) {
      +                    // We propagate TransientTransactionErrors to allow the state function to
      +                    // automatically be retried when TestData.runInsideTransaction=true
      +                    if (e.hasOwnProperty('errorLabels') &&
      +                        e.errorLabels.includes('TransientTransactionError')) {
      +                        throw e;
      +                    }
      +                    // dropIndex can cause queries to throw if these queries yield.
      +                    assertAlways.contains(e.code,
      +                                          [ErrorCodes.QueryPlanKilled, ErrorCodes.OperationFailed],
      +                                          'unexpected error code: ' + e.code + ': ' + e.message);
      +                }
      +            }
      +        },
      +
      +        readDocs: function readDocs(db, collName) {
      +            for (let i = 0; i < 5; ++i) {
      +                try {
      +                    let res = db[collName].findOne({value: this.docValue});
      +                    if (res !== null) {
      +                        assertAlways.eq(this.docValue, res.value);
      +                    }
      +                } catch (e) {
      +                    // We propagate TransientTransactionErrors to allow the state function to
      +                    // automatically be retried when TestData.runInsideTransaction=true
      +                    if (e.hasOwnProperty('errorLabels') &&
      +                        e.errorLabels.includes('TransientTransactionError')) {
      +                        throw e;
      +                    }
      +                    // dropIndex can cause queries to throw if these queries yield.
      +                    assertAlways.contains(e.code,
      +                                          [ErrorCodes.QueryPlanKilled, ErrorCodes.OperationFailed],
      +                                          'unexpected error code: ' + e.code + ': ' + e.message);
      +                }
      +            }
      +        },
      +
      +        deleteDocs: function deleteDocs(db, collName) {
      +            let indexToDelete = Math.floor(Math.random() * this.numIds);
      +            try {
      +                db[collName].deleteOne({_id: indexToDelete});
      +            } catch (e) {
      +                // We propagate TransientTransactionErrors to allow the state function to
      +                // automatically be retried when TestData.runInsideTransaction=true
      +                if (e.hasOwnProperty('errorLabels') &&
      +                    e.errorLabels.includes('TransientTransactionError')) {
      +                    throw e;
      +                }
      +                // dropIndex can cause queries to throw if these queries yield.
      +                assertAlways.contains(e.code,
      +                                      [ErrorCodes.QueryPlanKilled, ErrorCodes.OperationFailed],
      +                                      'unexpected error code: ' + e.code + ': ' + e.message);
      +            }
      +        },
      +
      +        createIndex: function createIndex(db, collName) {
      +            db[collName].createIndex({value: 1});
      +        },
      +
      +        createIdIndex: function createIdIndex(db, collName) {
      +            assertWhenOwnColl.commandWorked(db[collName].createIndex({_id: 1}));
      +        },
      +
      +        dropIndex: function dropIndex(db, collName) {
      +            db[collName].dropIndex({value: 1});
      +        },
      +
      +        dropCollection: function dropCollection(db, collName) {
      +            db[collName].drop();
      +            /*
      +            // The above drop forces this state to be run outside of a transaction, so it is fine to
      +            // ignore DuplicateKey errors here.
      +            for (let i = 0; i < this.numIds; i++) {
      +                const res = db[collName].insert({_id: i, value: this.docValue, num: 1});
      +                assertAlways.commandWorkedOrFailedWithCode(res, ErrorCodes.DuplicateKey);
      +            }*/
      +        }
      +
      +    };
      +
      +    const transitions = {
      +        init: {
      +            insertDocs: 0.10,
      +            updateDocs: 0.10,
      +            readDocs: 0.10,
      +            deleteDocs: 0.10,
      +            createIndex: 0.10,
      +            createIdIndex: 0.10,
      +            dropIndex: 0.10,
      +            dropCollection: 0.10
      +        },
      +        insertDocs: {
      +            insertDocs: 0.10,
      +            updateDocs: 0.10,
      +            readDocs: 0.10,
      +            deleteDocs: 0.10,
      +            createIndex: 0.10,
      +            createIdIndex: 0.10,
      +            dropIndex: 0.10,
      +            dropCollection: 0.30
      +        },
      +        updateDocs: {
      +            insertDocs: 0.10,
      +            updateDocs: 0.10,
      +            readDocs: 0.10,
      +            deleteDocs: 0.10,
      +            createIndex: 0.10,
      +            createIdIndex: 0.10,
      +            dropIndex: 0.10,
      +            dropCollection: 0.30
      +        },
      +        readDocs: {
      +            insertDocs: 0.10,
      +            updateDocs: 0.10,
      +            readDocs: 0.10,
      +            deleteDocs: 0.10,
      +            createIndex: 0.10,
      +            createIdIndex: 0.10,
      +            dropIndex: 0.10,
      +            dropCollection: 0.30
      +        },
      +        deleteDocs: {
      +            insertDocs: 0.10,
      +            updateDocs: 0.10,
      +            readDocs: 0.10,
      +            deleteDocs: 0.10,
      +            createIndex: 0.10,
      +            createIdIndex: 0.10,
      +            dropIndex: 0.10,
      +            dropCollection: 0.30
      +        },
      +        createIndex: {
      +            insertDocs: 0.10,
      +            updateDocs: 0.10,
      +            readDocs: 0.10,
      +            deleteDocs: 0.10,
      +            createIndex: 0.10,
      +            createIdIndex: 0.10,
      +            dropIndex: 0.10,
      +            dropCollection: 0.30
      +        },
      +        createIdIndex: {
      +            insertDocs: 0.10,
      +            updateDocs: 0.10,
      +            readDocs: 0.10,
      +            deleteDocs: 0.10,
      +            createIndex: 0.10,
      +            createIdIndex: 0.10,
      +            dropIndex: 0.10,
      +            dropCollection: 0.30
      +        },
      +        dropIndex: {
      +            insertDocs: 0.10,
      +            updateDocs: 0.10,
      +            readDocs: 0.10,
      +            deleteDocs: 0.10,
      +            createIndex: 0.10,
      +            createIdIndex: 0.10,
      +            dropIndex: 0.10,
      +            dropCollection: 0.30
      +        },
      +        dropCollection: {
      +            insertDocs: 0.10,
      +            updateDocs: 0.10,
      +            readDocs: 0.10,
      +            deleteDocs: 0.10,
      +            createIndex: 0.10,
      +            createIdIndex: 0.10,
      +            dropIndex: 0.10,
      +            dropCollection: 0.10
      +        }
      +    };
      +
      +    function setup(db, collName, cluster) {
      +        assertAlways.commandWorked(db.runCommand({create: collName}));
      +        for (let i = 0; i < this.numIds; i++) {
      +            const res = db[collName].insert({_id: i, value: this.docValue, num: 1});
      +            assertAlways.commandWorked(res);
      +            assert.eq(1, res.nInserted);
      +        }
      +    }
      +
      +    return {
      +        threadCount: 5,
      +        iterations: 10,
      +        startState: 'init',
      +        states: states,
      +        transitions: transitions,
      +        setup: setup,
      +        data: data,
      +    };
      +})();
      Show
      jstests/concurrency/fsm_workloads/CRUD_and_commands.js diff --git a/jstests/concurrency/fsm_workloads/CRUD_and_commands.js b/jstests/concurrency/fsm_workloads/CRUD_and_commands.js new file mode 100644 index 0000000000..6fc472dd93 --- /dev/null +++ b/jstests/concurrency/fsm_workloads/CRUD_and_commands.js @@ -0,0 +1,223 @@ +'use strict'; + +/** + * Perform CRUD operations, some of which may implicitly create collections. Also perform index + * creations which may implicitly create collections. + */ + +var $config = (function() { + const data = {numIds: 10, numDocsToInsertPerThread: 5, valueToBeInserted: 1, batchSize: 50}; + + const states = { + init: function init(db, collName) { + this.session = db.getMongo().startSession({causalConsistency: true}); + this.sessionDb = this.session.getDatabase(db.getName()); + this.docValue = "mydoc"; + }, + + insertDocs: function insertDocs(db, collName) { + for (let i = 0; i < 5; i++) { + const res = db[collName].insert({value: this.docValue, num: 1}); + assertWhenOwnColl.commandWorked(res); + assertWhenOwnColl.eq(1, res.nInserted); + } + }, + + updateDocs: function updateDocs(db, collName) { + for (let i = 0; i < 5; ++i) { + let indexToUpdate = Math.floor(Math.random() * this.numIds); + try { + assertWhenOwnColl.commandWorked(db[collName].update( + {_id: indexToUpdate}, {$inc: {num: 1}}, {upsert: true})); + } catch (e) { + // We propagate TransientTransactionErrors to allow the state function to + // automatically be retried when TestData.runInsideTransaction=true + if (e.hasOwnProperty('errorLabels') && + e.errorLabels.includes('TransientTransactionError')) { + throw e; + } + // dropIndex can cause queries to throw if these queries yield. + assertAlways.contains(e.code, + [ErrorCodes.QueryPlanKilled, ErrorCodes.OperationFailed], + 'unexpected error code: ' + e.code + ': ' + e.message); + } + } + }, + + readDocs: function readDocs(db, collName) { + for (let i = 0; i < 5; ++i) { + try { + let res = db[collName].findOne({value: this.docValue}); + if (res !== null) { + assertAlways.eq(this.docValue, res.value); + } + } catch (e) { + // We propagate TransientTransactionErrors to allow the state function to + // automatically be retried when TestData.runInsideTransaction=true + if (e.hasOwnProperty('errorLabels') && + e.errorLabels.includes('TransientTransactionError')) { + throw e; + } + // dropIndex can cause queries to throw if these queries yield. + assertAlways.contains(e.code, + [ErrorCodes.QueryPlanKilled, ErrorCodes.OperationFailed], + 'unexpected error code: ' + e.code + ': ' + e.message); + } + } + }, + + deleteDocs: function deleteDocs(db, collName) { + let indexToDelete = Math.floor(Math.random() * this.numIds); + try { + db[collName].deleteOne({_id: indexToDelete}); + } catch (e) { + // We propagate TransientTransactionErrors to allow the state function to + // automatically be retried when TestData.runInsideTransaction=true + if (e.hasOwnProperty('errorLabels') && + e.errorLabels.includes('TransientTransactionError')) { + throw e; + } + // dropIndex can cause queries to throw if these queries yield. + assertAlways.contains(e.code, + [ErrorCodes.QueryPlanKilled, ErrorCodes.OperationFailed], + 'unexpected error code: ' + e.code + ': ' + e.message); + } + }, + + createIndex: function createIndex(db, collName) { + db[collName].createIndex({value: 1}); + }, + + createIdIndex: function createIdIndex(db, collName) { + assertWhenOwnColl.commandWorked(db[collName].createIndex({_id: 1})); + }, + + dropIndex: function dropIndex(db, collName) { + db[collName].dropIndex({value: 1}); + }, + + dropCollection: function dropCollection(db, collName) { + db[collName].drop(); + /* + // The above drop forces this state to be run outside of a transaction, so it is fine to + // ignore DuplicateKey errors here. + for (let i = 0; i < this.numIds; i++) { + const res = db[collName].insert({_id: i, value: this.docValue, num: 1}); + assertAlways.commandWorkedOrFailedWithCode(res, ErrorCodes.DuplicateKey); + }*/ + } + + }; + + const transitions = { + init: { + insertDocs: 0.10, + updateDocs: 0.10, + readDocs: 0.10, + deleteDocs: 0.10, + createIndex: 0.10, + createIdIndex: 0.10, + dropIndex: 0.10, + dropCollection: 0.10 + }, + insertDocs: { + insertDocs: 0.10, + updateDocs: 0.10, + readDocs: 0.10, + deleteDocs: 0.10, + createIndex: 0.10, + createIdIndex: 0.10, + dropIndex: 0.10, + dropCollection: 0.30 + }, + updateDocs: { + insertDocs: 0.10, + updateDocs: 0.10, + readDocs: 0.10, + deleteDocs: 0.10, + createIndex: 0.10, + createIdIndex: 0.10, + dropIndex: 0.10, + dropCollection: 0.30 + }, + readDocs: { + insertDocs: 0.10, + updateDocs: 0.10, + readDocs: 0.10, + deleteDocs: 0.10, + createIndex: 0.10, + createIdIndex: 0.10, + dropIndex: 0.10, + dropCollection: 0.30 + }, + deleteDocs: { + insertDocs: 0.10, + updateDocs: 0.10, + readDocs: 0.10, + deleteDocs: 0.10, + createIndex: 0.10, + createIdIndex: 0.10, + dropIndex: 0.10, + dropCollection: 0.30 + }, + createIndex: { + insertDocs: 0.10, + updateDocs: 0.10, + readDocs: 0.10, + deleteDocs: 0.10, + createIndex: 0.10, + createIdIndex: 0.10, + dropIndex: 0.10, + dropCollection: 0.30 + }, + createIdIndex: { + insertDocs: 0.10, + updateDocs: 0.10, + readDocs: 0.10, + deleteDocs: 0.10, + createIndex: 0.10, + createIdIndex: 0.10, + dropIndex: 0.10, + dropCollection: 0.30 + }, + dropIndex: { + insertDocs: 0.10, + updateDocs: 0.10, + readDocs: 0.10, + deleteDocs: 0.10, + createIndex: 0.10, + createIdIndex: 0.10, + dropIndex: 0.10, + dropCollection: 0.30 + }, + dropCollection: { + insertDocs: 0.10, + updateDocs: 0.10, + readDocs: 0.10, + deleteDocs: 0.10, + createIndex: 0.10, + createIdIndex: 0.10, + dropIndex: 0.10, + dropCollection: 0.10 + } + }; + + function setup(db, collName, cluster) { + assertAlways.commandWorked(db.runCommand({create: collName})); + for (let i = 0; i < this.numIds; i++) { + const res = db[collName].insert({_id: i, value: this.docValue, num: 1}); + assertAlways.commandWorked(res); + assert.eq(1, res.nInserted); + } + } + + return { + threadCount: 5, + iterations: 10, + startState: 'init', + states: states, + transitions: transitions, + setup: setup, + data: data, + }; +})();

      I am adding a concurrency workload (included in steps to reproduce) as part of SERVER-44409 that, among other operations, performs updates with upsert:true concurrently with dropCollection. This workload can fail with the following error:
      Exec error resulting in state FAILURE :: caused by :: collection dropped. UUID dbaf99a7-24f5-4054-ac60-116b0f97a283
      I did not observe this error when the workload was run as part of our transactions override concurrency testing, which leads me to believe there is some window of time when locks do not conflict and somehow a collection can be dropped before the update occurs.

      I would expect these two operations to safely execute concurrently, since update should just create the collection if upsert is true.

            Assignee:
            backlog-query-execution [DO NOT USE] Backlog - Query Execution
            Reporter:
            maria.vankeulen@mongodb.com Maria van Keulen
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

              Created:
              Updated: