[SERVER-18787] printShardingStatus/sh.status should warn when "foreign" chunks are found Created: 02/Jun/15  Updated: 06/Dec/22  Resolved: 08/Nov/21

Status: Closed
Project: Core Server
Component/s: Diagnostics, Sharding
Affects Version/s: None
Fix Version/s: None

Type: Improvement Priority: Major - P3
Reporter: Kevin Pulo Assignee: [DO NOT USE] Backlog - Sharding EMEA
Resolution: Won't Do Votes: 0
Labels: neweng
Remaining Estimate: Not Specified
Time Spent: Not Specified
Original Estimate: Not Specified

Assigned Teams:
Sharding EMEA
Participants:

 Description   

printShardingStatus() / sh.status() give no indication when config.chunks contains documents that refer to collections that aren't in config.collections (with dropped: false).

Since this is an anomalous situation that will generally warrant further investigation into the state of the configDB, it would be very useful if sh.status() issued a warning when such chunks are found.

For example, I'm imagining something like:

sh.enableSharding("test")
sh.shardCollection("test.test", { _id: 1 } )
 
// mangle config db by adding a chunk for a non-existent collection
var chunk = dbs.config.chunks.find().next()
chunk._id = "test.test2-_id_MinKey"
chunk.ns = "test.test2"
db.getSiblingDB("config").chunks.insert(chunk)
 
// mangle config db by faking a collection drop
sh.shardCollection("test.test3", { _id: 1 } )
for (i = 0; i < 10; i++) sh.splitAt("test.test3", { _id: i } )
db.getSiblingDB("config").collections.update( { _id: "test.test3" }, { $set: { dropped: true } } )

 > sh.status()
 --- Sharding Status ---
   sharding version: {
         "_id" : 1,
         "minCompatibleVersion" : 5,
         "currentVersion" : 6,
         "clusterId" : ObjectId("556d787f313068f6d94b360d")
 }
   shards:
         {  "_id" : "shard01",  "host" : "genique:11112" )
   balancer:
         Currently enabled:  yes
         Currently running:  no
         Failed balancer rounds in last 5 attempts:  0
         Migration Results for the last 24 hours:
                 No recent migrations
   databases:
         {  "_id" : "admin",  "partitioned" : false,  "primary" : "config" }
         {  "_id" : "test",  "partitioned" : true,  "primary" : "shard01" }
                 test.test
                         shard key: { "_id" : 1 }
                         chunks:
                                 shard01 1
                         { "_id" : { "$minKey" : 1 } } -->> { "_id" : { "$maxKey" : 1 } } on : shard01 Timestamp(1, 0)
 
+  WARNING: Foreign chunks found:
+        { "ns" : "test.test3", "numChunks" : 11 }
+        { "ns" : "test.test2", "numChunks" : 1 }
+        Use sh.getForeignChunks() for the full list.
+
+> sh.getForeignChunks()
+{ "_id" : "test.test2-_id_MinKey", "lastmod" : Timestamp(1, 0), "lastmodEpoch" : ObjectId("556d78a9313068f6d94b361c"), "ns" : "test.test2", "min" : { "_id" : { "$minKey" : 1 } }, "max" : { "_id" : { "$maxKey" : 1 } }, "shard" : "shard01" }
+{ "_id" : "test.test3-_id_MinKey", "lastmod" : Timestamp(1, 1), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : { "$minKey" : 1 } }, "max" : { "_id" : 0 }, "shard" : "shard01" }
+{ "_id" : "test.test3-_id_0.0", "lastmod" : Timestamp(1, 3), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 0 }, "max" : { "_id" : 1 }, "shard" : "shard01" }
+{ "_id" : "test.test3-_id_1.0", "lastmod" : Timestamp(1, 5), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 1 }, "max" : { "_id" : 2 }, "shard" : "shard01" }
+{ "_id" : "test.test3-_id_2.0", "lastmod" : Timestamp(1, 7), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 2 }, "max" : { "_id" : 3 }, "shard" : "shard01" }
+{ "_id" : "test.test3-_id_3.0", "lastmod" : Timestamp(1, 9), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 3 }, "max" : { "_id" : 4 }, "shard" : "shard01" }
+{ "_id" : "test.test3-_id_4.0", "lastmod" : Timestamp(1, 11), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 4 }, "max" : { "_id" : 5 }, "shard" : "shard01" }
+{ "_id" : "test.test3-_id_5.0", "lastmod" : Timestamp(1, 13), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 5 }, "max" : { "_id" : 6 }, "shard" : "shard01" }
+{ "_id" : "test.test3-_id_6.0", "lastmod" : Timestamp(1, 15), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 6 }, "max" : { "_id" : 7 }, "shard" : "shard01" }
+{ "_id" : "test.test3-_id_7.0", "lastmod" : Timestamp(1, 17), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 7 }, "max" : { "_id" : 8 }, "shard" : "shard01" }
+{ "_id" : "test.test3-_id_8.0", "lastmod" : Timestamp(1, 19), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 8 }, "max" : { "_id" : 9 }, "shard" : "shard01" }
+{ "_id" : "test.test3-_id_9.0", "lastmod" : Timestamp(1, 20), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 9 }, "max" : { "_id" : { "$maxKey" : 1 } }, "shard" : "shard01" }

Generated from code along the lines of:

    var foreignCounts = configDB.chunks.aggregate(
        [
            {
                $match: {
                    ns: {
                        $nin: configDB.collections.find( { dropped: false }, { _id: 1 } ).map( function(x) { return x._id; } )
                    }
                }
            },
            {
                $group: {
                    _id: "$ns",
                    count: { $sum: 1 }
                }
            }
        ] ).map( function (x) { return { ns: x._id, numChunks: x.count }; } );
 
    if (foreignCounts.length > 0) {
        output( "\n  WARNING: Foreign chunks found:" );
        foreignCounts.forEach( function (x) { output( "\t" + tojson(x) ); } );
        output( "\tUse sh.getForeignChunks() for the full list." );
    }

sh.getForeignChunks = function (configDB) {
    if (configDB === undefined)
        configDB = db.getSisterDB('config')
 
	return configDB.chunks.find( {
            ns: {
                $nin: configDB.collections.find( { dropped: false }, { _id: 1 } ).map( function(x) { return x._id; } )
            }
        } ).sort( { ns: 1, min: 1 } );
};



 Comments   
Comment by Kaloian Manassiev [ 08/Nov/21 ]

We are going in a direction where we want people to never have to rely on direct reads/writes to the config collections. In such a world, we don't want to be enhancing pringShardingStatus even more.

Generated at Thu Feb 08 03:48:45 UTC 2024 using Jira 9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66.