Disallow compound wildcard indexes from being used as a shard key index

XMLWordPrintableJSON

    • Catalog and Routing
    • Fully Compatible
    • ALL
    • v8.2, v8.0, v7.0
    • Hide 
      // Documents containing the min value of a moveChunk will 
// get lost on a chunk migration.

import {ShardingTest} from "jstests/libs/shardingtest.js";

let s = new ShardingTest({name: "experimentColl", shards: 2, mongos: 1});
let shardedDb = s.getDB("test");
assert(s.adminCommand({enablesharding: "test", primaryShard: s.shard1.shardName}));

const experimentColl = shardedDb.experimentColl;

assert(experimentColl.drop())
assert.commandWorked(experimentColl.insert([
	{
		"_id" : 0,
		"a" : 0,
		"b" : 0
	},
	{
		"_id" : 1,
		"a" : 0,
		"b" : 10
	},
	{
		"_id" : 2,
		"a" : 0,
		"b" : -10
	}
]))

// We should have 3 documents
const resultsBeforeSharding = experimentColl.aggregate([]).toArray();
assert.eq(resultsBeforeSharding.length, 3)

// Shard with a wildcard component in the index
// Another index that repros the issue is {b: 1, '$**': -1} with {wildcardProjection: {a: 1}}
assert.commandWorked(experimentColl.createIndex({b: 1, 'a.$**': -1}));
assert(s.adminCommand({shardcollection: "test.experimentColl", key: {b: 1}}));

// Now split the and move the data between the shards
assert(s.adminCommand({split: "test.experimentColl", middle: {b: 0}}));
assert(s.adminCommand({
    moveChunk: "test.experimentColl",
    find: {b: 1},
    to: s.getOther(s.getPrimaryShard("test")).name,
    _waitForDelete: true
}));


const results = experimentColl.aggregate([]).toArray()
assert.eq(results.length, 3);
// finds 2 documents instead of 3

s.stop();

      // Documents that don't contain the shardKey may get 
// lost due to chunk migrations.

import {ShardingTest} from "jstests/libs/shardingtest.js";

let s = new ShardingTest({name: "experimentColl", shards: 2, mongos: 1});
let shardedDb = s.getDB("test");
assert(s.adminCommand({enablesharding: "test", primaryShard: s.shard0.shardName}));

const experimentColl = shardedDb.experimentColl;

assert(experimentColl.drop())
assert.commandWorked(
    experimentColl.insert([{"_id": 0, "x": 0}, {"_id": 1, "b": 10}, {"_id": 2, "b": -10}]))

// We should have 3 documents
const resultsBeforeSharding = experimentColl.aggregate([]).toArray();
assert.eq(resultsBeforeSharding.length, 3)

// Shard with a wildcard component in the index
assert.commandWorked(experimentColl.createIndex({b: 1, 'a.$**': 1}));
assert(s.adminCommand({shardcollection: "test.experimentColl", key: {b: 1}}));

assert(s.adminCommand({
    moveRange: "test.experimentColl",
    min: {b: MinKey},
    max: {b: MaxKey},
    toShard: s.shard1.shardName,
}));

const results = experimentColl.aggregate([]).toArray()
assert.eq(results.length, 3);
// finds 2 documents instead of 3

s.stop();
      Show
      // Documents containing the min value of a moveChunk will // get lost on a chunk migration. import {ShardingTest} from "jstests/libs/shardingtest.js" ; let s = new ShardingTest({name: "experimentColl" , shards: 2, mongos: 1}); let shardedDb = s.getDB( "test" ); assert (s.adminCommand({enablesharding: "test" , primaryShard: s.shard1.shardName})); const experimentColl = shardedDb.experimentColl; assert (experimentColl.drop()) assert .commandWorked(experimentColl.insert([ { "_id" : 0, "a" : 0, "b" : 0 }, { "_id" : 1, "a" : 0, "b" : 10 }, { "_id" : 2, "a" : 0, "b" : -10 } ])) // We should have 3 documents const resultsBeforeSharding = experimentColl.aggregate([]).toArray(); assert .eq(resultsBeforeSharding.length, 3) // Shard with a wildcard component in the index // Another index that repros the issue is {b: 1, '$**' : -1} with {wildcardProjection: {a: 1}} assert .commandWorked(experimentColl.createIndex({b: 1, 'a.$**' : -1})); assert (s.adminCommand({shardcollection: "test.experimentColl" , key: {b: 1}})); // Now split the and move the data between the shards assert (s.adminCommand({split: "test.experimentColl" , middle: {b: 0}})); assert (s.adminCommand({ moveChunk: "test.experimentColl" , find: {b: 1}, to: s.getOther(s.getPrimaryShard( "test" )).name, _waitForDelete: true })); const results = experimentColl.aggregate([]).toArray() assert .eq(results.length, 3); // finds 2 documents instead of 3 s.stop(); // Documents that don't contain the shardKey may get // lost due to chunk migrations. import {ShardingTest} from "jstests/libs/shardingtest.js" ; let s = new ShardingTest({name: "experimentColl" , shards: 2, mongos: 1}); let shardedDb = s.getDB( "test" ); assert (s.adminCommand({enablesharding: "test" , primaryShard: s.shard0.shardName})); const experimentColl = shardedDb.experimentColl; assert (experimentColl.drop()) assert .commandWorked( experimentColl.insert([{ "_id" : 0, "x" : 0}, { "_id" : 1, "b" : 10}, { "_id" : 2, "b" : -10}])) // We should have 3 documents const resultsBeforeSharding = experimentColl.aggregate([]).toArray(); assert .eq(resultsBeforeSharding.length, 3) // Shard with a wildcard component in the index assert .commandWorked(experimentColl.createIndex({b: 1, 'a.$**' : 1})); assert (s.adminCommand({shardcollection: "test.experimentColl" , key: {b: 1}})); assert (s.adminCommand({ moveRange: "test.experimentColl" , min: {b: MinKey}, max: {b: MaxKey}, toShard: s.shard1.shardName, })); const results = experimentColl.aggregate([]).toArray() assert .eq(results.length, 3); // finds 2 documents instead of 3 s.stop();
    • CAR Team 2025-09-01, CAR Team 2025-09-15, CAR Team 2025-09-29
    • 🟥 DDL
    • None
    • None
    • None
    • None
    • None
    • None

      Issue Status as of <CA send date>

      ISSUE DESCRIPTION AND IMPACT
      Compound wildcard indexes prefixed with the shard key can be incorrectly used as a shard key index to determine which documents to move during chunk migrations, causing some documents to be left behind on the donor shard and resulting in data loss.

      When a chunk migration uses a compound wildcard index as a shard key index, any documents that are missing all of the fields covered by that index will remain on the donor shard and won’t be migrated.
      Independently, if the compound wildcard index is defined with a descending wildcard field (for example, {skey: 1, 'a.$**': 1}), the chunk migration also leaves behind documents whose shard key is equal to the chunk’s min boundary.

      Left-behind documents may either be deleted by the range deleter (true data loss) or remain in a shard that doesn’t own them. If that same range of data is moved back later, those leftover documents can become visible again, leading to inconsistent results.

       

      DIAGNOSIS AND AFFECTED VERSIONS
      Sharded collections with compound wildcard indexes prefixed with the shard key may be affected on versions MongoDB 7.0.0-7.0.27, MongoDB 8.0.0-8.0.16, and MongoDB 8.2.0-8.2.1.

      To assess impact, identify sharded collections that have (or historically had) a compound wildcard index whose prefix is the shard key, especially where the balancer has been active or chunks were moved.

      1. The following steps explain how to verify that a sharded collection has a compound wildcard index prefixed with the shard key:
        List the collection’s indexes and review their key patterns. In mongosh, run the following to list all indexes for that collection: db.getSiblingDB("<db>").getCollection("<collection>").getIndexes().
      2. Identify compound wildcard indexes: look for an index key that includes $** plus at least one additional field (for example, {skey: 1, 'a.$**': 1}).
      3. Confirm the shard key prefix: the shard key field(s) must appear first in the index key pattern, in the same order as the shard key.
      4. If these conditions are met, the index is a compound wildcard index prefixed with the shard key.

       

      REMEDIATION AND WORKAROUNDS

      • Upgrade to a fixed version and keep the balancer disabled until all the remediation steps are completed. The fixed versions are MongoDB 7.0.28+ (i.e., any 7.0.Y that includes the fix), 8.0.17+, or 8.2.2+.
      • If dangling orphans are suspected (e.g., due to prior migrations using a compound wildcard index), use the provided remediation procedures/scripts to detect, recover, and clean up dangling orphans. Note that newer versions may surface such cases during chunk migrations due to SERVER-110953.
      • Because this fix removes compound wildcard indexes from the set of indexes that can support a shard key, some sharded collections might no longer have a valid shard key index after upgrade. Therefore, you’ll need to verify that every sharded collection has a valid shard key index. If a collection reports a MissingShardKeyIndex in the checkMetadataConsistency command, you’ll need to create a valid shard key index via a mongos connection. Additionally, if range deletions are blocked on shards that do not own chunks (log message 23765 “Unable to find range shard key index”), create the shard key index through direct shard connections on those shards to unblock the range deleter.
      • Finally, re-enable the balancer if it was enabled before the upgrade.

      Interim workarounds if you cannot upgrade immediately, you can either:

      • Disable the balancer and the range deleter (through the `disableResumableRangeDeleter` server parameter) for all sharded collections that have a compound wildcard index prefixed with the shard key, OR
      • Drop all the compound wildcard indexes that are prefixed with the shard key.

      —-----------------------------------------------------

       

      Original description:

      Some compound wildcard indexes can be used as the shard key index, when this is not supported and can lead to severe correctness issues. We should add validation to prevent this.

      In the repro attached, we see that when a collection is sharded and the shard key index has a wildcard component, documents can be missed in a full collection scan plan.

            Assignee:
            Silvia Surroca
            Reporter:
            Matt Boros
            Votes:
            0 Vote for this issue
            Watchers:
            26 Start watching this issue

              Created:
              Updated:
              Resolved: