Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-89668

Enable change streams / coll opts jstests for recordIdsReplicated:true

    • Type: Icon: Task Task
    • Resolution: Unresolved
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: None
    • Component/s: None
    • Labels:
      None
    • Storage Execution

      The diff below gets change stream jstests to work - assuming the every new collection created has recordIdsReplicated:true when the feature flag is turned on. Note that there might be more tests to be enabled by the time this ticket is done, so the diff below isn't exhaustive.

      However, since we opted for the route of adding a new builder that turns the default on (SERVER-89633), now though the jstests work on the new builder they don't work on the current all feature flags builder where recordIdsReplicated:false by default.

      diff --git a/jstests/aggregation/sources/out/out_preserve_coll_options.js b/jstests/aggregation/sources/out/out_preserve_coll_options.js
      index 80ca68e25d3..5095759af41 100644
      --- a/jstests/aggregation/sources/out/out_preserve_coll_options.js
      +++ b/jstests/aggregation/sources/out/out_preserve_coll_options.js
      @@ -20,6 +20,7 @@ const targetOptionsResponse =
           assert.commandWorked(db.runCommand({listCollections: 1, filter: {"name": targetName}}));
       const targetOptionsResults = new DBCommandCursor(db, targetOptionsResponse).toArray();
       assert.eq(targetOptionsResults.length, 1, targetOptionsResults);
      +delete targetOptionsResults[0].options.recordIdsReplicated;
       assert.eq({validationLevel: "moderate"}, targetOptionsResults[0].options, targetOptionsResults[0]);
       
       // Run $out pipeline.
      @@ -32,5 +33,6 @@ const targetOptionsResponseNew =
           assert.commandWorked(db.runCommand({listCollections: 1, filter: {"name": targetName}}));
       const targetOptionsResultsNew = new DBCommandCursor(db, targetOptionsResponseNew).toArray();
       assert.eq(targetOptionsResultsNew.length, 1, targetOptionsResultsNew);
      +delete targetOptionsResultsNew[0].options.recordIdsReplicated;
       assert.eq(
           {validationLevel: "moderate"}, targetOptionsResultsNew[0].options, targetOptionsResultsNew[0]);
      \ No newline at end of file
      diff --git a/jstests/change_streams/ddl_coll_mod_event.js b/jstests/change_streams/ddl_coll_mod_event.js
      index d812157524f..2ee411f772e 100644
      --- a/jstests/change_streams/ddl_coll_mod_event.js
      +++ b/jstests/change_streams/ddl_coll_mod_event.js
      @@ -7,6 +7,7 @@
        */
       import {ChangeStreamTest} from "jstests/libs/change_stream_util.js";
       import {FixtureHelpers} from "jstests/libs/fixture_helpers.js";
      +import {maybeAttachRecordIdsReplicated} from "jstests/libs/replicated_record_ids_utils.js";
       
       const testDB = db.getSiblingDB(jsTestName());
       
      @@ -68,7 +69,8 @@ function runTest(startChangeStream) {
                       ns: ns,
                       operationDescription: {validator: options.schemaValidator},
                       stateBeforeChange: {
      -                    collectionOptions: {uuid: getCollectionUuid(collName)},
      +                    collectionOptions:
      +                        maybeAttachRecordIdsReplicated({uuid: getCollectionUuid(collName)}),
                       }
                   });
               }
      @@ -86,12 +88,12 @@ function runTest(startChangeStream) {
                       ns: ns,
                       operationDescription: {validationLevel: newValidationLevel},
                       stateBeforeChange: {
      -                    collectionOptions: {
      +                    collectionOptions: maybeAttachRecordIdsReplicated({
                               uuid: getCollectionUuid(collName),
                               validator: options.schemaValidator,
                               validationLevel: options.validationLevel,
                               validationAction: options.validationAction
      -                    }
      +                    })
                       }
                   });
               }
      @@ -112,12 +114,12 @@ function runTest(startChangeStream) {
                       ns: ns,
                       operationDescription: {validationAction: newValidationAction},
                       stateBeforeChange: {
      -                    collectionOptions: {
      +                    collectionOptions: maybeAttachRecordIdsReplicated({
                               uuid: getCollectionUuid(collName),
                               validator: options.schemaValidator,
                               validationLevel: options.validationLevel,
      -                        validationAction: options.validationAction
      -                    }
      +                        validationAction: options.validationAction,
      +                    })
                       }
                   });
               }
      @@ -168,7 +170,8 @@ function runTest(startChangeStream) {
                       ns: ns,
                       operationDescription: toggleIndexHiddenOp,
                       stateBeforeChange: {
      -                    collectionOptions: {uuid: getCollectionUuid(collName)},
      +                    collectionOptions:
      +                        maybeAttachRecordIdsReplicated({uuid: getCollectionUuid(collName)}),
                           indexOptions: {hidden: !toggleIndexHiddenOp.index.hidden}
                       }
                   });
      @@ -183,7 +186,8 @@ function runTest(startChangeStream) {
                       ns: ns,
                       operationDescription: undoToggleIndexHiddenOp,
                       stateBeforeChange: {
      -                    collectionOptions: {uuid: getCollectionUuid(collName)},
      +                    collectionOptions:
      +                        maybeAttachRecordIdsReplicated({uuid: getCollectionUuid(collName)}),
                           indexOptions: {hidden: !undoToggleIndexHiddenOp.index.hidden}
                       }
                   });
      @@ -207,7 +211,8 @@ function runTest(startChangeStream) {
                           ns: ns,
                           operationDescription: modifyIndexExpireAfterSecondsOp,
                           stateBeforeChange: {
      -                        collectionOptions: {uuid: getCollectionUuid(collName)},
      +                        collectionOptions:
      +                            maybeAttachRecordIdsReplicated({uuid: getCollectionUuid(collName)}),
                               indexOptions: options
                           }
                       });
      diff --git a/jstests/change_streams/ddl_create_event.js b/jstests/change_streams/ddl_create_event.js
      index f18266e601b..6013cb75a35 100644
      --- a/jstests/change_streams/ddl_create_event.js
      +++ b/jstests/change_streams/ddl_create_event.js
      @@ -7,6 +7,7 @@
       import {assertChangeStreamEventEq, ChangeStreamTest} from "jstests/libs/change_stream_util.js";
       import {assertDropCollection} from "jstests/libs/collection_drop_recreate.js";
       import {FixtureHelpers} from "jstests/libs/fixture_helpers.js";
      +import {maybeAttachRecordIdsReplicated} from "jstests/libs/replicated_record_ids_utils.js";
       
       const testDB = db.getSiblingDB(jsTestName());
       
      @@ -48,7 +49,8 @@ function runTest(startChangeStream) {
           validateExpectedEventAndDropCollection({create: collName}, {
               operationType: "create",
               ns: ns,
      -        operationDescription: {idIndex: {v: 2, key: {_id: 1}, name: "_id_"}}
      +        operationDescription:
      +            maybeAttachRecordIdsReplicated({idIndex: {v: 2, key: {_id: 1}, name: "_id_"}})
           });
       
           // With implicit create collection through insert.
      @@ -57,7 +59,8 @@ function runTest(startChangeStream) {
           assertNextChangeEvent(cursor, {
               operationType: "create",
               ns: ns,
      -        operationDescription: {idIndex: {v: 2, key: {_id: 1}, name: "_id_"}}
      +        operationDescription:
      +            maybeAttachRecordIdsReplicated({idIndex: {v: 2, key: {_id: 1}, name: "_id_"}})
           });
           assertNextChangeEvent(
               cursor, {operationType: "insert", fullDocument: {_id: 0}, ns: ns, documentKey: {_id: 0}});
      @@ -65,17 +68,17 @@ function runTest(startChangeStream) {
       
           // With capped collection parameters.
           let expectedSize = 1000;
      -    validateExpectedEventAndDropCollection({create: collName, capped: true, size: 1000, max: 1000},
      -                                           {
      -                                               operationType: "create",
      -                                               ns: ns,
      -                                               operationDescription: {
      -                                                   idIndex: {v: 2, key: {_id: 1}, name: "_id_"},
      -                                                   capped: true,
      -                                                   size: expectedSize,
      -                                                   max: 1000
      -                                               }
      -                                           });
      +    validateExpectedEventAndDropCollection(
      +        {create: collName, capped: true, size: 1000, max: 1000}, {
      +            operationType: "create",
      +            ns: ns,
      +            operationDescription: maybeAttachRecordIdsReplicated({
      +                idIndex: {v: 2, key: {_id: 1}, name: "_id_"},
      +                capped: true,
      +                size: expectedSize,
      +                max: 1000
      +            })
      +        });
       
           // With wired tiger setting.
           const customWiredTigerSettings = {wiredTiger: {configString: "block_compressor=zlib"}};
      @@ -88,11 +91,11 @@ function runTest(startChangeStream) {
               {
                   operationType: "create",
                   ns: ns,
      -            operationDescription: {
      +            operationDescription: maybeAttachRecordIdsReplicated({
                       idIndex: {v: 2, key: {_id: 1}, name: "_id_"},
                       indexOptionDefaults: {storageEngine: customWiredTigerSettings},
                       storageEngine: customWiredTigerSettings
      -            }
      +            })
               });
       
           // With validator collection parameters.
      @@ -100,12 +103,12 @@ function runTest(startChangeStream) {
               {create: collName, validator: {a: 1}, validationLevel: "off", validationAction: "warn"}, {
                   operationType: "create",
                   ns: ns,
      -            operationDescription: {
      +            operationDescription: maybeAttachRecordIdsReplicated({
                       idIndex: {v: 2, key: {_id: 1}, name: "_id_"},
                       validator: {a: 1},
                       validationLevel: "off",
                       validationAction: "warn"
      -            }
      +            })
               });
       
           // With collation.
      @@ -124,10 +127,10 @@ function runTest(startChangeStream) {
           validateExpectedEventAndDropCollection({create: collName, collation: collation}, {
               operationType: "create",
               ns: ns,
      -        operationDescription: {
      +        operationDescription: maybeAttachRecordIdsReplicated({
                   collation: collation,
                   idIndex: {v: 2, key: {_id: 1}, name: "_id_", collation: collation}
      -        }
      +        })
           });
       
           // With clustered index.
      @@ -151,13 +154,16 @@ function runTest(startChangeStream) {
               {create: collName, idIndex: {v: 2, key: {_id: 1}, name: "ignored"}}, {
                   operationType: "create",
                   ns: ns,
      -            operationDescription: {
      +            operationDescription: maybeAttachRecordIdsReplicated({
                       idIndex: {v: 2, key: {_id: 1}, name: "_id_"},
      -            }
      +            })
               });
           validateExpectedEventAndDropCollection(
      -        {create: collName, idIndex: {v: 1, key: {_id: 1}, name: "new"}},
      -        {operationType: "create", ns: ns, operationDescription: {}});
      +        {create: collName, idIndex: {v: 1, key: {_id: 1}, name: "new"}}, {
      +            operationType: "create",
      +            ns: ns,
      +            operationDescription: maybeAttachRecordIdsReplicated({})
      +        });
       
           // Verify that the time-series create command does not produce an event.
           cursor = startChangeStream();
      @@ -175,9 +181,9 @@ function runTest(startChangeStream) {
               assertNextChangeEvent(cursor, {
                   operationType: "create",
                   ns,
      -            operationDescription: {
      +            operationDescription: maybeAttachRecordIdsReplicated({
                       idIndex: {v: 2, key: {_id: 1}, name: "_id_"},
      -            }
      +            })
               });
               assertDropCollection(testDB, collName);
           }
      diff --git a/jstests/change_streams/ddl_view_events.js b/jstests/change_streams/ddl_view_events.js
      index 1d048f225ca..a4c93944db8 100644
      --- a/jstests/change_streams/ddl_view_events.js
      +++ b/jstests/change_streams/ddl_view_events.js
      @@ -5,6 +5,7 @@
        */
       import {assertChangeStreamEventEq, ChangeStreamTest} from "jstests/libs/change_stream_util.js";
       import {assertDropCollection} from "jstests/libs/collection_drop_recreate.js";
      +import {maybeAttachRecordIdsReplicated} from "jstests/libs/replicated_record_ids_utils.js";
       
       const testDB = db.getSiblingDB(jsTestName());
       
      @@ -179,7 +180,8 @@ assert(event.collectionUUID, event);
       assertChangeStreamEventEq(event, {
           operationType: "create",
           ns: {db: dbName, coll: "view"},
      -    operationDescription: {idIndex: {v: 2, key: {_id: 1}, name: "_id_"}}
      +    operationDescription:
      +        maybeAttachRecordIdsReplicated({idIndex: {v: 2, key: {_id: 1}, name: "_id_"}})
       });
       
       // Change stream on a single collection does not produce view events.
      diff --git a/jstests/change_streams/resume_expanded_events.js b/jstests/change_streams/resume_expanded_events.js
      index 5ff8641cb83..6d064f1ade6 100644
      --- a/jstests/change_streams/resume_expanded_events.js
      +++ b/jstests/change_streams/resume_expanded_events.js
      @@ -11,6 +11,7 @@
        * ]
        */
       import {ChangeStreamTest} from "jstests/libs/change_stream_util.js";
      +import {maybeAttachRecordIdsReplicated} from "jstests/libs/replicated_record_ids_utils.js";
       
       const testDB = db.getSiblingDB(jsTestName());
       const collName = "coll1";
      @@ -36,7 +37,8 @@ function runTest(collNameForChangeStream) {
               expectedChanges: {
                   operationType: "create",
                   ns: ns,
      -            operationDescription: {idIndex: {v: 2, key: {_id: 1}, name: "_id_"}}
      +            operationDescription:
      +                maybeAttachRecordIdsReplicated({idIndex: {v: 2, key: {_id: 1}, name: "_id_"}})
               }
           })[0];
       
      diff --git a/jstests/change_streams/show_expanded_events.js b/jstests/change_streams/show_expanded_events.js
      index a3261b9550b..aa79a3634bf 100644
      --- a/jstests/change_streams/show_expanded_events.js
      +++ b/jstests/change_streams/show_expanded_events.js
      @@ -13,6 +13,7 @@ import {
           assertDropAndRecreateCollection,
           assertDropCollection,
       } from "jstests/libs/collection_drop_recreate.js";
      +import {maybeAttachRecordIdsReplicated} from "jstests/libs/replicated_record_ids_utils.js";
       
       const testDB = db.getSiblingDB(jsTestName());
       
      @@ -149,8 +150,8 @@ assertCreateCollection(testDB, renamedCollName);
       assertNextChangeEvent({
           ns: renamedNs,
           operationType: 'create',
      -    operationDescription: {idIndex: {v: 2, key: {_id: 1}, name: "_id_"}}
      -
      +    operationDescription:
      +        maybeAttachRecordIdsReplicated({idIndex: {v: 2, key: {_id: 1}, name: "_id_"}})
       });
       assertChangeEvent(() => assert.commandWorked(coll.renameCollection(renamedCollName, true)), {
           ns,
      diff --git a/jstests/core/catalog/list_collections_filter.js b/jstests/core/catalog/list_collections_filter.js
      index 1b743502a51..f7dd91eddd6 100644
      --- a/jstests/core/catalog/list_collections_filter.js
      +++ b/jstests/core/catalog/list_collections_filter.js
      @@ -44,6 +44,18 @@ function testListCollections(filter, expectedNames) {
               filter = {};
           }
       
      +    if (FeatureFlagUtil.isPresentAndEnabled(db, "RecordIdsReplicated")) {
      +        if (filter.options) {
      +            filter.options.recordIdsReplicated = true;
      +        } else if (filter["$and"]) {
      +            for (const pred of filter["$and"]) {
      +                if (pred.options) {
      +                    pred.options.recordIdsReplicated = true;
      +                }
      +            }
      +        }
      +    }
      +
           const cursor = new DBCommandCursor(mydb, mydb.runCommand("listCollections", {filter: filter}));
           function stripToName(result) {
               return result.name;
      diff --git a/jstests/core/query/internal_rename_if_options_and_indexes_match.js b/jstests/core/query/internal_rename_if_options_and_indexes_match.js
      index 2efeb4d9d6e..3afba925d59 100644
      --- a/jstests/core/query/internal_rename_if_options_and_indexes_match.js
      +++ b/jstests/core/query/internal_rename_if_options_and_indexes_match.js
      @@ -33,6 +33,9 @@ let commandObj = {
           indexes: [],
           collectionOptions: {uuid: optionsArray[0].info.uuid}
       };
      +if (optionsArray[0].options.recordIdsReplicated) {
      +    commandObj.collectionOptions.recordIdsReplicated = optionsArray[0].options.recordIdsReplicated;
      +}
       // Destination has an extra index.
       assert.commandFailedWithCode(adminDB.runCommand(commandObj), ErrorCodes.CommandFailed);
       
      @@ -65,4 +68,7 @@ commandObj.collectionOptions = {
           size: 256,
           max: 2,
       };
      +if (optionsArray[0].options.recordIdsReplicated) {
      +    commandObj.collectionOptions.recordIdsReplicated = optionsArray[0].options.recordIdsReplicated;
      +}
       assert.commandWorked(adminDB.runCommand(commandObj));
      \ No newline at end of file
      diff --git a/jstests/libs/replicated_record_ids_utils.js b/jstests/libs/replicated_record_ids_utils.js
      index 19dcf84eef1..2948c187770 100644
      --- a/jstests/libs/replicated_record_ids_utils.js
      +++ b/jstests/libs/replicated_record_ids_utils.js
      @@ -5,6 +5,17 @@ function getShowRecordIdsCursor(node, dbName, replicatedCollName) {
               [{"$project": {"recordId": {"$meta": "recordId"}, "document": "$$ROOT"}}]);
       }
       
      +import {FeatureFlagUtil} from "jstests/libs/feature_flag_util.js";
      +// Attaches the 'recordIdsReplicated:true' field to a give object. Useful for modifying
      +// existing tests that assert that a collection's options look a certain way.
      +// NOTE: modifies the passed in object.
      +export function maybeAttachRecordIdsReplicated(object) {
      +    if (FeatureFlagUtil.isPresentAndEnabled(db, "RecordIdsReplicated")) {
      +        object.recordIdsReplicated = true;
      +    }
      +    return object;
      +}
      +
       // Confirms data returned from a full collection scan on 'replicatedCollName', with the '$recordId'
       // field included for each document, yields the same results across all nodes.
       export function validateShowRecordIdReplicatesAcrossNodes(nodes, dbName, replicatedCollName) {
      @@ -98,7 +109,7 @@ export function testPreservingRecordIdsDuringInitialSync(
               const initialSyncNode = replTest.add({
                   rsConfig: {priority: 0},
                   setParameter: {
      -                logComponentVerbosity: tojsononeline({replication: 0, storage: 0}),
      +                logComponentVerbosity: tojsononeline({replication: 5, storage: 5}),
                       initialSyncMethod: initSyncMethod
                   }
               });
      
      

            Assignee:
            backlog-server-execution [DO NOT USE] Backlog - Storage Execution Team
            Reporter:
            vishnu.kaushik@mongodb.com Vishnu Kaushik
            Votes:
            0 Vote for this issue
            Watchers:
            1 Start watching this issue

              Created:
              Updated: