Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-3159

Re-adding member in replica set fails

    • Type: Icon: Bug Bug
    • Resolution: Done
    • Priority: Icon: Minor - P4 Minor - P4
    • None
    • Affects Version/s: 1.6.5
    • Component/s: Replication
    • Labels:
      None
    • Environment:
      Replica Set on Ubuntu 10.4 LTS on Amazon EC2
    • Linux

      Removing and re-adding a member of a replica set seems to work:

      > rs.remove("m2:27017")

      { "ok" : 1 }

      > rs.status();
      {
      "set" : "wm1",
      "date" : "Fri May 27 2011 13:56:52 GMT+0000 (UTC)",
      "myState" : 1,
      "members" : [

      { "_id" : 1, "name" : "web2:27017", "health" : 1, "state" : 7, "uptime" : 2, "lastHeartbeat" : "Fri May 27 2011 13:56:52 GMT+0000 (UTC)" }

      ,

      { "_id" : 2, "name" : "web3:27017", "health" : 1, "state" : 7, "uptime" : 2, "lastHeartbeat" : "Fri May 27 2011 13:56:52 GMT+0000 (UTC)" }

      ,

      { "_id" : 3, "name" : "m5:27017", "health" : 1, "state" : 1, "self" : true }

      ,

      { "_id" : 4, "name" : "m3:27017", "health" : 1, "state" : 2, "uptime" : 2, "lastHeartbeat" : "Fri May 27 2011 13:56:52 GMT+0000 (UTC)" }

      ],
      "ok" : 1
      }

      > rs.add("m2:27017")

      { "ok" : 1 }

      > rs.status();
      {
      "set" : "wm1",
      "date" : "Fri May 27 2011 13:57:01 GMT+0000 (UTC)",
      "myState" : 1,
      "members" : [

      { "_id" : 1, "name" : "web2:27017", "health" : 1, "state" : 7, "uptime" : 11, "lastHeartbeat" : "Fri May 27 2011 13:57:00 GMT+0000 (UTC)" }

      ,

      { "_id" : 2, "name" : "web3:27017", "health" : 1, "state" : 7, "uptime" : 11, "lastHeartbeat" : "Fri May 27 2011 13:57:00 GMT+0000 (UTC)" }

      ,

      { "_id" : 3, "name" : "m5:27017", "health" : 1, "state" : 1, "self" : true }

      ,
      {
      "_id" : 4,
      "name" : "m3:27017",
      "health" : 1,
      "state" : 2,
      "uptime" : 11,
      "lastHeartbeat" : "Fri May 27 2011 13:57:00 GMT+0000 (UTC)",
      "errmsg" : "syncThread: 13106 nextSafe():

      { $err: \"cursor dropped during query\", code: 13338 }

      "
      },

      { "_id" : 5, "name" : "m2:27017", "health" : 1, "state" : 4, "uptime" : 2, "lastHeartbeat" : "Fri May 27 2011 13:56:59 GMT+0000 (UTC)" }

      ],
      "ok" : 1
      }

      But the logs on the re-added node show this repeating message:

      Fri May 27 14:03:14 [rs Manager] replSet error unexpected exception in haveNewConfig() : 0 assertion db/repl/rs.cpp:315
      Fri May 27 14:03:14 [rs Manager] replSet error fatal, stopping replication
      Fri May 27 14:03:15 [initandlisten] connection accepted from 10.254.238.86:57200 #107
      Fri May 27 14:03:16 [conn107] end connection 10.254.238.86:57200
      Fri May 27 14:03:16 [rs Manager] replset msgReceivedNewConfig version: version: 7
      Fri May 27 14:03:17 [rs Manager] replSet info saving a newer config version to local.system.replset
      Fri May 27 14:03:23 [rs Manager] replSet 0 5
      Fri May 27 14:03:23 [rs Manager] Assertion failure false db/repl/rs.cpp 315
      0x534a81 0x54163f 0x66a6aa 0x66e97e 0x66f5a6 0x695073 0x549473 0x547821 0x546f83 0x52b028 0x83a4b0 0x7f097a4939ca 0x7f0979a4270d
      /usr/bin/mongod(_ZN5mongo12sayDbContextEPKc+0xb1) [0x534a81]
      /usr/bin/mongod(_ZN5mongo8assertedEPKcS1_j+0x10f) [0x54163f]
      /usr/bin/mongod(_ZN5mongo11ReplSetImpl14initFromConfigERNS_13ReplSetConfigEb+0x17a) [0x66a6aa]
      /usr/bin/mongod(_ZN5mongo7ReplSet13haveNewConfigERNS_13ReplSetConfigEb+0xfe) [0x66e97e]
      /usr/bin/mongod(_ZN5mongo7Manager20msgReceivedNewConfigENS_7BSONObjE+0x256) [0x66f5a6]
      /usr/bin/mongod(_ZN5boost6detail8function26void_function_obj_invoker0INS_3_bi6bind_tIvNS_4_mfi3mf1IvN5mongo7ManagerENS7_7BSONObjEEENS3_5list2INS3_5valueIPS8_EENSC_IS9_EEEEEEvE6invokeERNS1_15function_bufferE+0x63) [0x695073]
      /usr/bin/mongod(_ZNK5boost9function0IvEclEv+0x243) [0x549473]
      /usr/bin/mongod(_ZN5mongo4task6Server6doWorkEv+0x141) [0x547821]
      /usr/bin/mongod(_ZN5mongo4task4Task3runEv+0x33) [0x546f83]
      /usr/bin/mongod(_ZN5mongo13BackgroundJob3thrEv+0x88) [0x52b028]
      /usr/bin/mongod(thread_proxy+0x80) [0x83a4b0]
      /lib/libpthread.so.0(+0x69ca) [0x7f097a4939ca]
      /lib/libc.so.6(clone+0x6d) [0x7f0979a4270d]

            Assignee:
            kristina Kristina Chodorow (Inactive)
            Reporter:
            skion Pieter Ennes
            Votes:
            0 Vote for this issue
            Watchers:
            2 Start watching this issue

              Created:
              Updated:
              Resolved: