Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-22273

Mongos not work correctly

    • Type: Icon: Bug Bug
    • Resolution: Done
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: None
    • Component/s: Sharding
    • None
    • ALL

      We deploy sharded cluster between multi locations(Site or Data center).

      issue: Mongos don't return any thing, and It show this message

      [thread1] Error: error: { "$err" : "Operation timed out", "code" : 50 } :
      

      I describe all details about this sharded cluster at below:

      1. mongodb version: 3.2.1
      2. QTY of locations(or Data center): 4 location to the names (thr, ifn, mhd, bnd)
      3. The location "thr" is main datacenter, Then the thr must be majority on replica set.
      4. We deploy the replica set for configsvr, and We put 4 members into thr, 4 into mhd, 4 into ifn and one member into bnd.
      5. The thr is majority, Then 3 of members is votes=1 and other is zero
      6. Other locations(mhd,ifn), They have just one member for votes=1
      7. the bnd location just one secondery with votes=0
      8. each location have one replica set(3 members) for sharding.
      9. We use tag aware for replica set
      10. We have 4 mongos, 4 replica set(3 member for two replica set, 2member for other replica set), 1 replica set configsvr(13 member)

      The result of rs.config() from configsvr:

      {
      	"_id" : "iran_cfg",
      	"version" : 6,
      	"configsvr" : true,
      	"protocolVersion" : NumberLong(1),
      	"members" : [
      		{
      			"_id" : 1,
      			"host" : "thr-cfg01:37017",
      			"arbiterOnly" : false,
      			"buildIndexes" : true,
      			"hidden" : false,
      			"priority" : 3,
      			"tags" : {
      				
      			},
      			"slaveDelay" : NumberLong(0),
      			"votes" : 1
      		},
      		{
      			"_id" : 2,
      			"host" : "thr-cfg02:37017",
      			"arbiterOnly" : false,
      			"buildIndexes" : true,
      			"hidden" : false,
      			"priority" : 2,
      			"tags" : {
      				
      			},
      			"slaveDelay" : NumberLong(0),
      			"votes" : 1
      		},
      		{
      			"_id" : 3,
      			"host" : "thr-cfg03:37017",
      			"arbiterOnly" : false,
      			"buildIndexes" : true,
      			"hidden" : false,
      			"priority" : 1,
      			"tags" : {
      				
      			},
      			"slaveDelay" : NumberLong(0),
      			"votes" : 1
      		},
      		{
      			"_id" : 4,
      			"host" : "thr-cfg04:37017",
      			"arbiterOnly" : false,
      			"buildIndexes" : true,
      			"hidden" : false,
      			"priority" : 0,
      			"tags" : {
      				
      			},
      			"slaveDelay" : NumberLong(0),
      			"votes" : 0
      		},
      		{
      			"_id" : 5,
      			"host" : "mhd-cfg01:37017",
      			"arbiterOnly" : false,
      			"buildIndexes" : true,
      			"hidden" : false,
      			"priority" : 0,
      			"tags" : {
      				
      			},
      			"slaveDelay" : NumberLong(0),
      			"votes" : 0
      		},
      		{
      			"_id" : 6,
      			"host" : "mhd-cfg02:37017",
      			"arbiterOnly" : false,
      			"buildIndexes" : true,
      			"hidden" : false,
      			"priority" : 0,
      			"tags" : {
      				
      			},
      			"slaveDelay" : NumberLong(0),
      			"votes" : 0
      		},
      		{
      			"_id" : 7,
      			"host" : "mhd-cfg03:37017",
      			"arbiterOnly" : false,
      			"buildIndexes" : true,
      			"hidden" : false,
      			"priority" : 1,
      			"tags" : {
      				
      			},
      			"slaveDelay" : NumberLong(0),
      			"votes" : 1
      		},
      		{
      			"_id" : 8,
      			"host" : "mhd-cfg04:37017",
      			"arbiterOnly" : false,
      			"buildIndexes" : true,
      			"hidden" : false,
      			"priority" : 0,
      			"tags" : {
      				
      			},
      			"slaveDelay" : NumberLong(0),
      			"votes" : 0
      		},
      		{
      			"_id" : 9,
      			"host" : "ifn-cfg01:37017",
      			"arbiterOnly" : false,
      			"buildIndexes" : true,
      			"hidden" : false,
      			"priority" : 0,
      			"tags" : {
      				
      			},
      			"slaveDelay" : NumberLong(0),
      			"votes" : 0
      		},
      		{
      			"_id" : 10,
      			"host" : "ifn-cfg02:37017",
      			"arbiterOnly" : false,
      			"buildIndexes" : true,
      			"hidden" : false,
      			"priority" : 0,
      			"tags" : {
      				
      			},
      			"slaveDelay" : NumberLong(0),
      			"votes" : 0
      		},
      		{
      			"_id" : 11,
      			"host" : "ifn-cfg03:37017",
      			"arbiterOnly" : false,
      			"buildIndexes" : true,
      			"hidden" : false,
      			"priority" : 1,
      			"tags" : {
      				
      			},
      			"slaveDelay" : NumberLong(0),
      			"votes" : 1
      		},
      		{
      			"_id" : 12,
      			"host" : "ifn-cfg04:37017",
      			"arbiterOnly" : false,
      			"buildIndexes" : true,
      			"hidden" : false,
      			"priority" : 0,
      			"tags" : {
      				
      			},
      			"slaveDelay" : NumberLong(0),
      			"votes" : 0
      		},
      		{
      			"_id" : 13,
      			"host" : "bnd-db03:37017",
      			"arbiterOnly" : false,
      			"buildIndexes" : true,
      			"hidden" : false,
      			"priority" : 0,
      			"tags" : {
      				
      			},
      			"slaveDelay" : NumberLong(0),
      			"votes" : 0
      		}
      	],
      	"settings" : {
      		"chainingAllowed" : true,
      		"heartbeatIntervalMillis" : 2000,
      		"heartbeatTimeoutSecs" : 10,
      		"electionTimeoutMillis" : 10000,
      		"getLastErrorModes" : {
      			
      		},
      		"getLastErrorDefaults" : {
      			"w" : 1,
      			"wtimeout" : 0
      		}
      	}
      }
      

      We have 4 mongos, But most of mongos not work correctly, and When I send command into router, my command(sh.status(), sh.add, show collections) will be freeze or crash.

      The config file of all configsvr:

      # mongod.conf
      
      # for documentation of all options, see:
      #   http://docs.mongodb.org/manual/reference/configuration-options/
      
      # Where and how to store data.
      storage:
        dbPath: /var/lib/mongodb
        journal:
          enabled: true
      #  engine:
      #  mmapv1:
      #  wiredTiger:
      
      # where to write logging data.
      systemLog:
        destination: file
        verbosity: 5
        traceAllExceptions: true
        logRotate: rename
        logAppend: true
        component:
            sharding:
               verbosity: 5
            command:
               verbosity: 5
            replication:
               verbosity: 5
            network:
               verbosity: 5
      
      
        path: /var/log/mongodb/mongod.log
      
      # network interfaces
      net:
        port: 37017
      
      sharding:
         clusterRole: configsvr
      
      replication:
         replSetName: iran_cfg
      

      The config file of shardsvr:

      # mongod.conf
      
      # for documentation of all options, see:
      #   http://docs.mongodb.org/manual/reference/configuration-options/
      
      # Where and how to store data.
      storage:
        dbPath: /var/lib/mongodb
        journal:
          enabled: true
      #  engine:
      #  mmapv1:
      #  wiredTiger:
      
      # where to write logging data.
      systemLog:
        destination: file
        verbosity: 5
        traceAllExceptions: true
        logRotate: rename
        logAppend: true
        component:
            sharding:
               verbosity: 5
            command:
               verbosity: 5
            replication:
               verbosity: 5
            network:
               verbosity: 5
      
        path: /var/log/mongodb/mongod.log
      
      # network interfaces
      net:
        port: 27018
      
      sharding:
         clusterRole: shardsvr
      
      # thr,mhd,ifn, ...
      replication:
         replSetName: bnd
      
      # processManagement:
      #    fork: true
      

      one of mongos config file:

      # mongod.conf
      
      # for documentation of all options, see:
      #   http://docs.mongodb.org/manual/reference/configuration-options/
      
      # Where and how to store data.
      #  engine:
      #  mmapv1:
      #  wiredTiger:
      
      # where to write logging data.
      systemLog:
        destination: file
        traceAllExceptions: true
        logRotate: rename
        quiet : true
        logAppend: true
        component:
            network:
               verbosity: 5
        path: /var/log/mongodb/mongod.log
      
      sharding:
          configDB: "iran_cfg/thr-cfg01:37017,thr-cfg02:37017,mhd-cfg03:37017,ifn-cfg03:37017"
          autoSplit: true
      
      net:
          port: 27017
      

      The result from sh.status():

      --- Sharding Status --- 
        sharding version: {
      	"_id" : 1,
      	"minCompatibleVersion" : 5,
      	"currentVersion" : 6,
      	"clusterId" : ObjectId("56a0f7da64fe06bcccb03350")
      }
        shards:
      	{  "_id" : "bnd",  "host" : "bnd/bnd-db01:27018,bnd-db02:27018" }
      	{  "_id" : "ifn",  "host" : "ifn/ifn-db01:27018,ifn-db02:27018,ifn-db03:27018" }
      	{  "_id" : "mhd",  "host" : "mhd/mhd-db01:27018,mhd-db02:27018,mhd-db03:27018" }
      	{  "_id" : "thr",  "host" : "thr/thr-db01:27018,thr-db02:27018,thr-db03:27018" }
        active mongoses:
      	{  "_id" : "db04-thr-srv:27017",  "ping" : ISODate("2016-01-22T12:29:02.648Z"),  "up" : NumberLong(320),  "waiting" : false,  "mongoVersion" : "3.2.1" }
      	{  "_id" : "db04-ifn-srv:27017",  "ping" : ISODate("2016-01-22T12:28:05.566Z"),  "up" : NumberLong(75752),  "waiting" : false,  "mongoVersion" : "3.2.1" }
        balancer:
      	Currently enabled:  yes
      	Currently running:  no
      	Failed balancer rounds in last 5 attempts:  5
      	Last reported error:  could not get updated shard list from config server due to ExceededTimeLimit Operation timed out
      	Time of Reported error:  Fri Jan 22 2016 15:57:15 GMT+0330 (IRST)
      	Migration Results for the last 24 hours: 
      		No recent migrations
        databases:
      

      I'm sorry for too busy information for this issue.

            Assignee:
            ramon.fernandez@mongodb.com Ramon Fernandez Marina
            Reporter:
            Ali.Hallaji Ali Hallaji
            Votes:
            0 Vote for this issue
            Watchers:
            7 Start watching this issue

              Created:
              Updated:
              Resolved: