[SERVER-9283] lastHeartbeatRecv flaps between invalid and valid dates - MongoDB Jira

XML

Word

Printable

JSON

Type: Bug
Status: Closed
Priority: Major - P3
Resolution: Fixed
Affects Version/s: 2.4.1, 2.5.0
Fix Version/s: 2.4.5, 2.5.1
Component/s: Replication
Labels:
None
Environment:
build info: Darwin anden.local 11.4.2 Darwin Kernel Version 11.4.2: Thu Aug 23 16:25:48 PDT 2012; root:xnu-1699.32.7~1/RELEASE_X86_64 x86_64 BOOST_LIB_VERSION=1_49
git version: 29bc5063bd232aa1664511d4486936d1f13e60b6

Every once in a while the lastHeartbeatRecv date stamp gets screwed up while initialising a replicaset.

Note: This is completely fresh replicaset with no data inserted yet

anden(mongod-2.5.0-pre-)[PRIMARY/failover] test> rs.status()

  "set": "failover",

  "date": ISODate("2013-04-08T12:03:12-0700"),

  "myState": 1,

  "members": [

      "_id": 0,

      "name": "anden.local:1337",

      "health": 1,

      "state": 1,

      "stateStr": "PRIMARY",

      "uptime": 86,

      "optime": {

        "t": 1365447776,

        "i": 1

},

      "optimeDate": ISODate("2013-04-08T12:02:56-0700"),

      "self": true

},

      "_id": 1,

      "name": "anden.local:1338",

      "health": 1,

      "state": 5,

      "stateStr": "STARTUP2",

      "uptime": 16,

      "optime": {

        "t": 0,

        "i": 0

},

      "optimeDate": ISODate("1969-12-31T16:00:00-0800"),

      "lastHeartbeat": ISODate("2013-04-08T12:03:12-0700"),

      "lastHeartbeatRecv": ISODate("1969-12-31T16:00:00-0800"),

      "pingMs": 0,

      "lastHeartbeatMessage": "initial sync need a member to be primary or secondary to do our initial sync"

],

  "ok": 1

anden(mongod-2.5.0-pre-)[PRIMARY/failover] test> rs.status()

  "set": "failover",

  "date": ISODate("2013-04-08T12:03:13-0700"),

  "myState": 1,

  "members": [

      "_id": 0,

      "name": "anden.local:1337",

      "health": 1,

      "state": 1,

      "stateStr": "PRIMARY",

      "uptime": 87,

      "optime": {

        "t": 1365447776,

        "i": 1

},

      "optimeDate": ISODate("2013-04-08T12:02:56-0700"),

      "self": true

},

      "_id": 1,

      "name": "anden.local:1338",

      "health": 1,

      "state": 5,

      "stateStr": "STARTUP2",

      "uptime": 17,

      "optime": {

        "t": 0,

        "i": 0

},

      "optimeDate": ISODate("1969-12-31T16:00:00-0800"),

      "lastHeartbeat": ISODate("2013-04-08T12:03:12-0700"),

      "lastHeartbeatRecv": ISODate("2013-04-08T12:03:13-0700"),

      "pingMs": 0,

      "lastHeartbeatMessage": "initial sync need a member to be primary or secondary to do our initial sync"

],

  "ok": 1

anden(mongod-2.5.0-pre-)[PRIMARY/failover] test> rs.status()

  "set": "failover",

  "date": ISODate("2013-04-08T12:03:13-0700"),

  "myState": 1,

  "members": [

      "_id": 0,

      "name": "anden.local:1337",

      "health": 1,

      "state": 1,

      "stateStr": "PRIMARY",

      "uptime": 87,

      "optime": {

        "t": 1365447776,

        "i": 1

},

      "optimeDate": ISODate("2013-04-08T12:02:56-0700"),

      "self": true

},

      "_id": 1,

      "name": "anden.local:1338",

      "health": 1,

      "state": 5,

      "stateStr": "STARTUP2",

      "uptime": 17,

      "optime": {

        "t": 0,

        "i": 0

},

      "optimeDate": ISODate("1969-12-31T16:00:00-0800"),

      "lastHeartbeat": ISODate("2013-04-08T12:03:12-0700"),

      "lastHeartbeatRecv": ISODate("2013-04-08T12:03:13-0700"),

      "pingMs": 0,

      "lastHeartbeatMessage": "initial sync need a member to be primary or secondary to do our initial sync"

],

  "ok": 1

anden(mongod-2.5.0-pre-)[PRIMARY/failover] test> rs.status()

  "set": "failover",

  "date": ISODate("2013-04-08T12:03:14-0700"),

  "myState": 1,

  "members": [

      "_id": 0,

      "name": "anden.local:1337",

      "health": 1,

      "state": 1,

      "stateStr": "PRIMARY",

      "uptime": 88,

      "optime": {

        "t": 1365447776,

        "i": 1

},

      "optimeDate": ISODate("2013-04-08T12:02:56-0700"),

      "self": true

},

      "_id": 1,

      "name": "anden.local:1338",

      "health": 1,

      "state": 5,

      "stateStr": "STARTUP2",

      "uptime": 18,

      "optime": {

        "t": 0,

        "i": 0

},

      "optimeDate": ISODate("1969-12-31T16:00:00-0800"),

      "lastHeartbeat": ISODate("2013-04-08T12:03:12-0700"),

      "lastHeartbeatRecv": ISODate("2013-04-08T12:03:13-0700"),

      "pingMs": 0,

      "lastHeartbeatMessage": "initial sync need a member to be primary or secondary to do our initial sync"

],

  "ok": 1

anden(mongod-2.5.0-pre-)[PRIMARY/failover] test> rs.status()

  "set": "failover",

  "date": ISODate("2013-04-08T12:03:14-0700"),

  "myState": 1,

  "members": [

      "_id": 0,

      "name": "anden.local:1337",

      "health": 1,

      "state": 1,

      "stateStr": "PRIMARY",

      "uptime": 88,

      "optime": {

        "t": 1365447776,

        "i": 1

},

      "optimeDate": ISODate("2013-04-08T12:02:56-0700"),

      "self": true

},

      "_id": 1,

      "name": "anden.local:1338",

      "health": 1,

      "state": 5,

      "stateStr": "STARTUP2",

      "uptime": 18,

      "optime": {

        "t": 0,

        "i": 0

},

      "optimeDate": ISODate("1969-12-31T16:00:00-0800"),

      "lastHeartbeat": ISODate("2013-04-08T12:03:14-0700"),

      "lastHeartbeatRecv": ISODate("1969-12-31T16:00:00-0800"),

      "pingMs": 0,

      "lastHeartbeatMessage": "initial sync need a member to be primary or secondary to do our initial sync"

],

  "ok": 1

is related to

SERVER-9730 Asymmetrical network partition can cause the election of two PRIMARY nodes

SERVER-9756 network partition where minority was primary and stayed primary

SERVER-9765 Two primaries should cause the earlier-elected one to step down

related to

SERVER-11280 ReplSetHealthPollTask::down uses m.lastHeartbeatRecv, which is not properly set

Participants:: auto, Hannes Magnusson, Jenny Duckett, Matt Campbell, Randolph Tan