printShardingStatus/sh.status should warn when "foreign" chunks are found

XMLWordPrintableJSON

    • Type: Improvement
    • Resolution: Won't Do
    • Priority: Major - P3
    • None
    • Affects Version/s: None
    • Component/s: Diagnostics, Sharding
    • Sharding EMEA
    • None
    • None
    • None
    • None
    • None
    • None
    • None

      printShardingStatus() / sh.status() give no indication when config.chunks contains documents that refer to collections that aren't in config.collections (with dropped: false).

      Since this is an anomalous situation that will generally warrant further investigation into the state of the configDB, it would be very useful if sh.status() issued a warning when such chunks are found.

      For example, I'm imagining something like:

      sh.enableSharding("test")
      sh.shardCollection("test.test", { _id: 1 } )
      
      // mangle config db by adding a chunk for a non-existent collection
      var chunk = dbs.config.chunks.find().next()
      chunk._id = "test.test2-_id_MinKey"
      chunk.ns = "test.test2"
      db.getSiblingDB("config").chunks.insert(chunk)
      
      // mangle config db by faking a collection drop
      sh.shardCollection("test.test3", { _id: 1 } )
      for (i = 0; i < 10; i++) sh.splitAt("test.test3", { _id: i } )
      db.getSiblingDB("config").collections.update( { _id: "test.test3" }, { $set: { dropped: true } } )
      
      Unable to find source-code formatter for language: diff. Available languages are: actionscript, ada, applescript, bash, c, c#, c++, cpp, css, erlang, go, groovy, haskell, html, java, javascript, js, json, lua, none, nyan, objc, perl, php, python, r, rainbow, ruby, scala, sh, sql, swift, visualbasic, xml, yaml
       > sh.status()
       --- Sharding Status ---
         sharding version: {
               "_id" : 1,
               "minCompatibleVersion" : 5,
               "currentVersion" : 6,
               "clusterId" : ObjectId("556d787f313068f6d94b360d")
       }
         shards:
               {  "_id" : "shard01",  "host" : "genique:11112" )
         balancer:
               Currently enabled:  yes
               Currently running:  no
               Failed balancer rounds in last 5 attempts:  0
               Migration Results for the last 24 hours:
                       No recent migrations
         databases:
               {  "_id" : "admin",  "partitioned" : false,  "primary" : "config" }
               {  "_id" : "test",  "partitioned" : true,  "primary" : "shard01" }
                       test.test
                               shard key: { "_id" : 1 }
                               chunks:
                                       shard01 1
                               { "_id" : { "$minKey" : 1 } } -->> { "_id" : { "$maxKey" : 1 } } on : shard01 Timestamp(1, 0)
       
      +  WARNING: Foreign chunks found:
      +        { "ns" : "test.test3", "numChunks" : 11 }
      +        { "ns" : "test.test2", "numChunks" : 1 }
      +        Use sh.getForeignChunks() for the full list.
      +
      +> sh.getForeignChunks()
      +{ "_id" : "test.test2-_id_MinKey", "lastmod" : Timestamp(1, 0), "lastmodEpoch" : ObjectId("556d78a9313068f6d94b361c"), "ns" : "test.test2", "min" : { "_id" : { "$minKey" : 1 } }, "max" : { "_id" : { "$maxKey" : 1 } }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_MinKey", "lastmod" : Timestamp(1, 1), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : { "$minKey" : 1 } }, "max" : { "_id" : 0 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_0.0", "lastmod" : Timestamp(1, 3), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 0 }, "max" : { "_id" : 1 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_1.0", "lastmod" : Timestamp(1, 5), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 1 }, "max" : { "_id" : 2 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_2.0", "lastmod" : Timestamp(1, 7), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 2 }, "max" : { "_id" : 3 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_3.0", "lastmod" : Timestamp(1, 9), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 3 }, "max" : { "_id" : 4 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_4.0", "lastmod" : Timestamp(1, 11), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 4 }, "max" : { "_id" : 5 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_5.0", "lastmod" : Timestamp(1, 13), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 5 }, "max" : { "_id" : 6 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_6.0", "lastmod" : Timestamp(1, 15), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 6 }, "max" : { "_id" : 7 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_7.0", "lastmod" : Timestamp(1, 17), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 7 }, "max" : { "_id" : 8 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_8.0", "lastmod" : Timestamp(1, 19), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 8 }, "max" : { "_id" : 9 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_9.0", "lastmod" : Timestamp(1, 20), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 9 }, "max" : { "_id" : { "$maxKey" : 1 } }, "shard" : "shard01" }
      

      Generated from code along the lines of:

          var foreignCounts = configDB.chunks.aggregate(
              [
                  {
                      $match: {
                          ns: {
                              $nin: configDB.collections.find( { dropped: false }, { _id: 1 } ).map( function(x) { return x._id; } )
                          }
                      }
                  },
                  {
                      $group: {
                          _id: "$ns",
                          count: { $sum: 1 }
                      }
                  }
              ] ).map( function (x) { return { ns: x._id, numChunks: x.count }; } );
      
          if (foreignCounts.length > 0) {
              output( "\n  WARNING: Foreign chunks found:" );
              foreignCounts.forEach( function (x) { output( "\t" + tojson(x) ); } );
              output( "\tUse sh.getForeignChunks() for the full list." );
          }
      
      sh.getForeignChunks = function (configDB) {
          if (configDB === undefined)
              configDB = db.getSisterDB('config')
      
      	return configDB.chunks.find( {
                  ns: {
                      $nin: configDB.collections.find( { dropped: false }, { _id: 1 } ).map( function(x) { return x._id; } )
                  }
              } ).sort( { ns: 1, min: 1 } );
      };
      

            Assignee:
            [DO NOT USE] Backlog - Sharding EMEA
            Reporter:
            Kevin Pulo
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

              Created:
              Updated:
              Resolved: