Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-94655

checkMetadataConsistency doesn't detect config.collections and shard local catalog metadata mismatch

    • Type: Icon: Bug Bug
    • Resolution: Gone away
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: None
    • Component/s: None
    • Catalog and Routing
    • ALL
    • Hide
      (function() {
      'use strict';
      
      var st = new ShardingTest({mongos: 1, shards: 2, other: {enableBalancer: false}});
      
      const zoneName = 'zone';
      const dbName = 'db';
      const collName = 'coll';
      const shard0 = st.shard0.shardName;
      const shard1 = st.shard1.shardName;
      const sKey = 'x';
      
      assert.commandWorked(st.s.adminCommand({enableSharding: dbName, primaryShard: shard0}));
      
      const coll = st.s.getDB(dbName)[collName];
      assert.commandWorked(st.s.adminCommand({addShardToZone: shard0, zone: zoneName}));
      
      // Ensure cache for the database is warmed up on the config server
      assert.commandWorked(st.configRS.getPrimary().adminCommand({_flushDatabaseCacheUpdates: dbName}));
      
      // Create collection with one chunk on shard 0
      assert.commandWorked(st.s.adminCommand({shardCollection: coll.getFullName(), key: {sKey: 1}}));
      
      assert(st.shard0.getCollection(coll.getFullName()).exists(),
             "Collection does not exist on shard0 despite the shard has chunks and is database primary");
      assert(
          !st.shard1.getCollection(coll.getFullName()).exists(),
          "Collection exists on shard1 even but shard1 is not db primary nor has chunks for the collection");
      assert(coll.exists());
      
      // Move the database primary, the config server does not invalidate the cache
      assert.commandWorked(st.s.adminCommand({movePrimary: dbName, to: shard1}))
      
      assert.commandWorked(st.s.adminCommand({
          reshardCollection: coll.getFullName(),
          key: {sKey: 1, "newField": 1},
          zones: [{
              min: {sKey: MinKey(), newField: MinKey()},
              max: {sKey: MaxKey(), newField: MaxKey()},
              zone: zoneName,
          }]
      }));
      
      assert(st.shard0.getCollection(coll.getFullName()).exists(),
             "Collection does not exist on shard0 despite the shard has chunks");
      
      let res = assert.commandWorked(st.s.adminCommand({checkMetadataConsistency: 1}));
      assert.neq(res.cursor.firstBatch.length, 0);
      jsTest.log("Logging checkMetadataConsistency");
      jsTest.log(res);
      assert(st.shard1.getCollection(coll.getFullName()).exists(),
             "Collection does not exist on shard1 despite the shard is database primary");
      assert(coll.exists());
      
      st.stop();
      })();
      
      
      Show
      (function() { 'use strict' ; var st = new ShardingTest({mongos: 1, shards: 2, other: {enableBalancer: false }}); const zoneName = 'zone' ; const dbName = 'db' ; const collName = 'coll' ; const shard0 = st.shard0.shardName; const shard1 = st.shard1.shardName; const sKey = 'x' ; assert .commandWorked(st.s.adminCommand({enableSharding: dbName, primaryShard: shard0})); const coll = st.s.getDB(dbName)[collName]; assert .commandWorked(st.s.adminCommand({addShardToZone: shard0, zone: zoneName})); // Ensure cache for the database is warmed up on the config server assert .commandWorked(st.configRS.getPrimary().adminCommand({_flushDatabaseCacheUpdates: dbName})); // Create collection with one chunk on shard 0 assert .commandWorked(st.s.adminCommand({shardCollection: coll.getFullName(), key: {sKey: 1}})); assert (st.shard0.getCollection(coll.getFullName()).exists(), "Collection does not exist on shard0 despite the shard has chunks and is database primary" ); assert ( !st.shard1.getCollection(coll.getFullName()).exists(), "Collection exists on shard1 even but shard1 is not db primary nor has chunks for the collection" ); assert (coll.exists()); // Move the database primary, the config server does not invalidate the cache assert .commandWorked(st.s.adminCommand({movePrimary: dbName, to: shard1})) assert .commandWorked(st.s.adminCommand({ reshardCollection: coll.getFullName(), key: {sKey: 1, "newField" : 1}, zones: [{ min: {sKey: MinKey(), newField: MinKey()}, max: {sKey: MaxKey(), newField: MaxKey()}, zone: zoneName, }] })); assert (st.shard0.getCollection(coll.getFullName()).exists(), "Collection does not exist on shard0 despite the shard has chunks" ); let res = assert .commandWorked(st.s.adminCommand({checkMetadataConsistency: 1})); assert .neq(res.cursor.firstBatch.length, 0); jsTest.log( "Logging checkMetadataConsistency" ); jsTest.log(res); assert (st.shard1.getCollection(coll.getFullName()).exists(), "Collection does not exist on shard1 despite the shard is database primary" ); assert (coll.exists()); st.stop(); })();
    • CAR Team 2024-09-30

      While testing impact/remediation of CA-130 it was found that checkMetadataConsistency command didn't return the inconsistent metadata for the resharded collection. Reproducer is attached - please note that the commit ID used for reproduction is 65bc345ba01d03826f3261a1db15f37b05ac939c. More details on the bug are present here in SERVER-86622

            Assignee:
            pol.pinol@mongodb.com Pol Pinol
            Reporter:
            abdul.qadeer@mongodb.com Abdul Qadeer
            Votes:
            0 Vote for this issue
            Watchers:
            7 Start watching this issue

              Created:
              Updated:
              Resolved: