Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-18787

printShardingStatus/sh.status should warn when "foreign" chunks are found

    XMLWordPrintableJSON

Details

    • Icon: Improvement Improvement
    • Resolution: Won't Do
    • Icon: Major - P3 Major - P3
    • None
    • None
    • Diagnostics, Sharding
    • Sharding EMEA

    Description

      printShardingStatus() / sh.status() give no indication when config.chunks contains documents that refer to collections that aren't in config.collections (with dropped: false).

      Since this is an anomalous situation that will generally warrant further investigation into the state of the configDB, it would be very useful if sh.status() issued a warning when such chunks are found.

      For example, I'm imagining something like:

      sh.enableSharding("test")
      sh.shardCollection("test.test", { _id: 1 } )
       
      // mangle config db by adding a chunk for a non-existent collection
      var chunk = dbs.config.chunks.find().next()
      chunk._id = "test.test2-_id_MinKey"
      chunk.ns = "test.test2"
      db.getSiblingDB("config").chunks.insert(chunk)
       
      // mangle config db by faking a collection drop
      sh.shardCollection("test.test3", { _id: 1 } )
      for (i = 0; i < 10; i++) sh.splitAt("test.test3", { _id: i } )
      db.getSiblingDB("config").collections.update( { _id: "test.test3" }, { $set: { dropped: true } } )
      

       > sh.status()
       --- Sharding Status ---
         sharding version: {
               "_id" : 1,
               "minCompatibleVersion" : 5,
               "currentVersion" : 6,
               "clusterId" : ObjectId("556d787f313068f6d94b360d")
       }
         shards:
               {  "_id" : "shard01",  "host" : "genique:11112" )
         balancer:
               Currently enabled:  yes
               Currently running:  no
               Failed balancer rounds in last 5 attempts:  0
               Migration Results for the last 24 hours:
                       No recent migrations
         databases:
               {  "_id" : "admin",  "partitioned" : false,  "primary" : "config" }
               {  "_id" : "test",  "partitioned" : true,  "primary" : "shard01" }
                       test.test
                               shard key: { "_id" : 1 }
                               chunks:
                                       shard01 1
                               { "_id" : { "$minKey" : 1 } } -->> { "_id" : { "$maxKey" : 1 } } on : shard01 Timestamp(1, 0)
       
      +  WARNING: Foreign chunks found:
      +        { "ns" : "test.test3", "numChunks" : 11 }
      +        { "ns" : "test.test2", "numChunks" : 1 }
      +        Use sh.getForeignChunks() for the full list.
      +
      +> sh.getForeignChunks()
      +{ "_id" : "test.test2-_id_MinKey", "lastmod" : Timestamp(1, 0), "lastmodEpoch" : ObjectId("556d78a9313068f6d94b361c"), "ns" : "test.test2", "min" : { "_id" : { "$minKey" : 1 } }, "max" : { "_id" : { "$maxKey" : 1 } }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_MinKey", "lastmod" : Timestamp(1, 1), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : { "$minKey" : 1 } }, "max" : { "_id" : 0 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_0.0", "lastmod" : Timestamp(1, 3), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 0 }, "max" : { "_id" : 1 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_1.0", "lastmod" : Timestamp(1, 5), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 1 }, "max" : { "_id" : 2 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_2.0", "lastmod" : Timestamp(1, 7), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 2 }, "max" : { "_id" : 3 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_3.0", "lastmod" : Timestamp(1, 9), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 3 }, "max" : { "_id" : 4 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_4.0", "lastmod" : Timestamp(1, 11), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 4 }, "max" : { "_id" : 5 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_5.0", "lastmod" : Timestamp(1, 13), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 5 }, "max" : { "_id" : 6 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_6.0", "lastmod" : Timestamp(1, 15), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 6 }, "max" : { "_id" : 7 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_7.0", "lastmod" : Timestamp(1, 17), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 7 }, "max" : { "_id" : 8 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_8.0", "lastmod" : Timestamp(1, 19), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 8 }, "max" : { "_id" : 9 }, "shard" : "shard01" }
      +{ "_id" : "test.test3-_id_9.0", "lastmod" : Timestamp(1, 20), "lastmodEpoch" : ObjectId("556d861d313068f6d94b38cf"), "ns" : "test.test3", "min" : { "_id" : 9 }, "max" : { "_id" : { "$maxKey" : 1 } }, "shard" : "shard01" }
      

      Generated from code along the lines of:

          var foreignCounts = configDB.chunks.aggregate(
              [
                  {
                      $match: {
                          ns: {
                              $nin: configDB.collections.find( { dropped: false }, { _id: 1 } ).map( function(x) { return x._id; } )
                          }
                      }
                  },
                  {
                      $group: {
                          _id: "$ns",
                          count: { $sum: 1 }
                      }
                  }
              ] ).map( function (x) { return { ns: x._id, numChunks: x.count }; } );
       
          if (foreignCounts.length > 0) {
              output( "\n  WARNING: Foreign chunks found:" );
              foreignCounts.forEach( function (x) { output( "\t" + tojson(x) ); } );
              output( "\tUse sh.getForeignChunks() for the full list." );
          }
      

      sh.getForeignChunks = function (configDB) {
          if (configDB === undefined)
              configDB = db.getSisterDB('config')
       
      	return configDB.chunks.find( {
                  ns: {
                      $nin: configDB.collections.find( { dropped: false }, { _id: 1 } ).map( function(x) { return x._id; } )
                  }
              } ).sort( { ns: 1, min: 1 } );
      };
      

      Attachments

        Activity

          People

            backlog-server-sharding-emea [DO NOT USE] Backlog - Sharding EMEA
            kevin.pulo@mongodb.com Kevin Pulo
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved: