cwd [/home/gregorv/Workspaces/10Gen/mongo] num procs:220 removing: /data/db/sconsTests//local.ns removing: /data/db/sconsTests//mongod.lock removing: /data/db/sconsTests//local.0 Wed Jan 16 18:10:13.820 [initandlisten] MongoDB starting : pid=5242 port=27999 dbpath=/data/db/sconsTests/ 64-bit host=nuwen Wed Jan 16 18:10:13.820 [initandlisten] Wed Jan 16 18:10:13.820 [initandlisten] ** NOTE: This is a development version (2.3.3-pre-) of MongoDB. Wed Jan 16 18:10:13.820 [initandlisten] ** Not recommended for production. Wed Jan 16 18:10:13.820 [initandlisten] Wed Jan 16 18:10:13.820 [initandlisten] db version v2.3.3-pre-, pdfile version 4.5 Wed Jan 16 18:10:13.820 [initandlisten] git version: b8c0b6ae71fb6c076101fa25ef915021ea26e156 Wed Jan 16 18:10:13.820 [initandlisten] build info: Linux nuwen 3.5.0-21-generic #32-Ubuntu SMP Tue Dec 11 18:51:59 UTC 2012 x86_64 BOOST_LIB_VERSION=1_49 Wed Jan 16 18:10:13.820 [initandlisten] allocator: tcmalloc Wed Jan 16 18:10:13.820 [initandlisten] options: { dbpath: "/data/db/sconsTests/", nopreallocj: true, port: 27999, setParameter: [ "enableTestCommands=1" ] } Wed Jan 16 18:10:13.882 [initandlisten] journal dir=/data/db/sconsTests/journal Wed Jan 16 18:10:13.882 [initandlisten] recover : no journal files present, no recovery needed Wed Jan 16 18:10:13.925 [FileAllocator] allocating new datafile /data/db/sconsTests/local.ns, filling with zeroes... Wed Jan 16 18:10:13.925 [FileAllocator] creating directory /data/db/sconsTests/_tmp Wed Jan 16 18:10:14.108 [FileAllocator] done allocating datafile /data/db/sconsTests/local.ns, size: 16MB, took 0.15 secs Wed Jan 16 18:10:14.108 [FileAllocator] allocating new datafile /data/db/sconsTests/local.0, filling with zeroes... Wed Jan 16 18:10:14.625 [FileAllocator] done allocating datafile /data/db/sconsTests/local.0, size: 64MB, took 0.517 secs Wed Jan 16 18:10:14.626 [initandlisten] command local.$cmd command: { create: "startup_log", size: 10485760, capped: true } ntoreturn:1 keyUpdates:0 reslen:37 701ms Wed Jan 16 18:10:14.627 [initandlisten] waiting for connections on port 27999 Wed Jan 16 18:10:14.627 [websvr] admin web console waiting for connections on port 28999 running /home/gregorv/Workspaces/10Gen/mongo/mongod --port 27999 --dbpath /data/db/sconsTests/ --setParameter enableTestCommands=1 --nopreallocj ******************************************* Test : sync_change_source.js ... Command : /home/gregorv/Workspaces/10Gen/mongo/mongo --port 27999 --authenticationMechanism MONGO-CR /home/gregorv/Workspaces/10Gen/mongo/jstests/replsets/sync_change_source.js --eval TestData = new Object();TestData.testPath = "/home/gregorv/Workspaces/10Gen/mongo/jstests/replsets/sync_change_source.js";TestData.testFile = "sync_change_source.js";TestData.testName = "sync_change_source";TestData.noJournal = false;TestData.noJournalPrealloc = true;TestData.auth = false;TestData.keyFile = null;TestData.keyFileData = null; Date : Wed Jan 16 18:10:14 2013 MongoDB shell version: 2.3.3-pre- connecting to: 127.0.0.1:27999/test Wed Jan 16 18:10:14.831 [initandlisten] connection accepted from 127.0.0.1:53686 #2 (1 connection now open) null Running: sudo bash -c tc qdisc del root dev lo > /tmp/output Wed Jan 16 18:10:14.836 shell: started program sudo bash -c tc qdisc del root dev lo > /tmp/output Running: sudo bash -c tc qdisc add dev lo handle 1: root htb > /tmp/output Wed Jan 16 18:10:21.566 shell: started program sudo bash -c tc qdisc add dev lo handle 1: root htb > /tmp/output Running: sudo bash -c tc class add dev lo parent 1: classid 1:1 htb rate 1000Mbps > /tmp/output Wed Jan 16 18:10:21.580 shell: started program sudo bash -c tc class add dev lo parent 1: classid 1:1 htb rate 1000Mbps > /tmp/output Running: sudo bash -c tc class add dev lo parent 1:1 classid 1:10 htb rate 1bps ceil 1000Mbps > /tmp/output Wed Jan 16 18:10:21.593 shell: started program sudo bash -c tc class add dev lo parent 1:1 classid 1:10 htb rate 1bps ceil 1000Mbps > /tmp/output ReplSetTest Starting Set ReplSetTest n is : 0 ReplSetTest n: 0 ports: [ 31000, 31001, 31002 ] 31000 number { "useHostName" : true, "oplogSize" : 40, "keyFile" : undefined, "port" : 31000, "noprealloc" : "", "smallfiles" : "", "rest" : "", "replSet" : "testReplSet", "dbpath" : "$set-$node", "useIP" : true, "restart" : undefined, "pathOpts" : { "node" : 0, "set" : "testReplSet" } } ReplSetTest Starting.... Using Local IP for host: 127.0.0.2 Resetting db path '/data/db/testReplSet-0' Wed Jan 16 18:10:21.873 shell: started program /home/gregorv/Workspaces/10Gen/mongo/mongod --oplogSize 40 --port 31000 --noprealloc --smallfiles --rest --replSet testReplSet --dbpath /data/db/testReplSet-0 --nopreallocj --bind_ip 127.0.0.2 --setParameter enableTestCommands=1 m31000| note: noprealloc may hurt performance in many applications m31000| Wed Jan 16 18:10:21.896 [initandlisten] MongoDB starting : pid=5284 port=31000 dbpath=/data/db/testReplSet-0 64-bit host=nuwen m31000| Wed Jan 16 18:10:21.896 [initandlisten] m31000| Wed Jan 16 18:10:21.896 [initandlisten] ** NOTE: This is a development version (2.3.3-pre-) of MongoDB. m31000| Wed Jan 16 18:10:21.896 [initandlisten] ** Not recommended for production. m31000| Wed Jan 16 18:10:21.896 [initandlisten] m31000| Wed Jan 16 18:10:21.896 [initandlisten] db version v2.3.3-pre-, pdfile version 4.5 m31000| Wed Jan 16 18:10:21.896 [initandlisten] git version: b8c0b6ae71fb6c076101fa25ef915021ea26e156 m31000| Wed Jan 16 18:10:21.896 [initandlisten] build info: Linux nuwen 3.5.0-21-generic #32-Ubuntu SMP Tue Dec 11 18:51:59 UTC 2012 x86_64 BOOST_LIB_VERSION=1_49 m31000| Wed Jan 16 18:10:21.896 [initandlisten] allocator: tcmalloc m31000| Wed Jan 16 18:10:21.896 [initandlisten] options: { bind_ip: "127.0.0.2", dbpath: "/data/db/testReplSet-0", noprealloc: true, nopreallocj: true, oplogSize: 40, port: 31000, replSet: "testReplSet", rest: true, setParameter: [ "enableTestCommands=1" ], smallfiles: true } m31000| Wed Jan 16 18:10:21.988 [initandlisten] journal dir=/data/db/testReplSet-0/journal m31000| Wed Jan 16 18:10:21.988 [initandlisten] recover : no journal files present, no recovery needed m31000| Wed Jan 16 18:10:22.038 [FileAllocator] allocating new datafile /data/db/testReplSet-0/local.ns, filling with zeroes... m31000| Wed Jan 16 18:10:22.039 [FileAllocator] creating directory /data/db/testReplSet-0/_tmp m31000| Wed Jan 16 18:10:22.230 [FileAllocator] done allocating datafile /data/db/testReplSet-0/local.ns, size: 16MB, took 0.149 secs m31000| Wed Jan 16 18:10:22.230 [FileAllocator] allocating new datafile /data/db/testReplSet-0/local.0, filling with zeroes... m31000| Wed Jan 16 18:10:22.371 [FileAllocator] done allocating datafile /data/db/testReplSet-0/local.0, size: 16MB, took 0.141 secs m31000| Wed Jan 16 18:10:22.373 [initandlisten] command local.$cmd command: { create: "startup_log", size: 10485760, capped: true } ntoreturn:1 keyUpdates:0 reslen:37 334ms m31000| Wed Jan 16 18:10:22.373 [initandlisten] waiting for connections on port 31000 m31000| Wed Jan 16 18:10:22.373 [websvr] admin web console waiting for connections on port 32000 m31000| Wed Jan 16 18:10:22.374 [rsStart] replSet can't get local.system.replset config from self or any seed (EMPTYCONFIG) m31000| Wed Jan 16 18:10:22.374 [rsStart] replSet info you may need to run replSetInitiate -- rs.initiate() in the shell -- if that is not already done m31000| Wed Jan 16 18:10:22.477 [initandlisten] connection accepted from 127.0.0.1:53396 #1 (1 connection now open) [ connection to 127.0.0.2:31000 ] ReplSetTest n is : 1 ReplSetTest n: 1 ports: [ 31000, 31001, 31002 ] 31001 number { "useHostName" : true, "oplogSize" : 40, "keyFile" : undefined, "port" : 31001, "noprealloc" : "", "smallfiles" : "", "rest" : "", "replSet" : "testReplSet", "dbpath" : "$set-$node", "useIP" : true, "restart" : undefined, "pathOpts" : { "node" : 1, "set" : "testReplSet" } } ReplSetTest Starting.... Using Local IP for host: 127.0.0.3 Resetting db path '/data/db/testReplSet-1' Wed Jan 16 18:10:22.536 shell: started program /home/gregorv/Workspaces/10Gen/mongo/mongod --oplogSize 40 --port 31001 --noprealloc --smallfiles --rest --replSet testReplSet --dbpath /data/db/testReplSet-1 --nopreallocj --bind_ip 127.0.0.3 --setParameter enableTestCommands=1 m31001| note: noprealloc may hurt performance in many applications m31001| Wed Jan 16 18:10:22.563 [initandlisten] MongoDB starting : pid=5335 port=31001 dbpath=/data/db/testReplSet-1 64-bit host=nuwen m31001| Wed Jan 16 18:10:22.563 [initandlisten] m31001| Wed Jan 16 18:10:22.563 [initandlisten] ** NOTE: This is a development version (2.3.3-pre-) of MongoDB. m31001| Wed Jan 16 18:10:22.563 [initandlisten] ** Not recommended for production. m31001| Wed Jan 16 18:10:22.563 [initandlisten] m31001| Wed Jan 16 18:10:22.563 [initandlisten] db version v2.3.3-pre-, pdfile version 4.5 m31001| Wed Jan 16 18:10:22.563 [initandlisten] git version: b8c0b6ae71fb6c076101fa25ef915021ea26e156 m31001| Wed Jan 16 18:10:22.563 [initandlisten] build info: Linux nuwen 3.5.0-21-generic #32-Ubuntu SMP Tue Dec 11 18:51:59 UTC 2012 x86_64 BOOST_LIB_VERSION=1_49 m31001| Wed Jan 16 18:10:22.563 [initandlisten] allocator: tcmalloc m31001| Wed Jan 16 18:10:22.563 [initandlisten] options: { bind_ip: "127.0.0.3", dbpath: "/data/db/testReplSet-1", noprealloc: true, nopreallocj: true, oplogSize: 40, port: 31001, replSet: "testReplSet", rest: true, setParameter: [ "enableTestCommands=1" ], smallfiles: true } m31001| Wed Jan 16 18:10:22.622 [initandlisten] journal dir=/data/db/testReplSet-1/journal m31001| Wed Jan 16 18:10:22.623 [initandlisten] recover : no journal files present, no recovery needed m31001| Wed Jan 16 18:10:22.665 [FileAllocator] allocating new datafile /data/db/testReplSet-1/local.ns, filling with zeroes... m31001| Wed Jan 16 18:10:22.665 [FileAllocator] creating directory /data/db/testReplSet-1/_tmp m31001| Wed Jan 16 18:10:22.848 [FileAllocator] done allocating datafile /data/db/testReplSet-1/local.ns, size: 16MB, took 0.15 secs m31001| Wed Jan 16 18:10:22.848 [FileAllocator] allocating new datafile /data/db/testReplSet-1/local.0, filling with zeroes... m31001| Wed Jan 16 18:10:22.998 [FileAllocator] done allocating datafile /data/db/testReplSet-1/local.0, size: 16MB, took 0.149 secs m31001| Wed Jan 16 18:10:22.999 [initandlisten] command local.$cmd command: { create: "startup_log", size: 10485760, capped: true } ntoreturn:1 keyUpdates:0 reslen:37 334ms m31001| Wed Jan 16 18:10:22.999 [initandlisten] waiting for connections on port 31001 m31001| Wed Jan 16 18:10:23.000 [websvr] admin web console waiting for connections on port 32001 m31001| Wed Jan 16 18:10:23.002 [rsStart] replSet can't get local.system.replset config from self or any seed (EMPTYCONFIG) m31001| Wed Jan 16 18:10:23.002 [rsStart] replSet info you may need to run replSetInitiate -- rs.initiate() in the shell -- if that is not already done m31001| Wed Jan 16 18:10:23.138 [initandlisten] connection accepted from 127.0.0.1:56516 #1 (1 connection now open) [ connection to 127.0.0.2:31000, connection to 127.0.0.3:31001 ] ReplSetTest n is : 2 ReplSetTest n: 2 ports: [ 31000, 31001, 31002 ] 31002 number { "useHostName" : true, "oplogSize" : 40, "keyFile" : undefined, "port" : 31002, "noprealloc" : "", "smallfiles" : "", "rest" : "", "replSet" : "testReplSet", "dbpath" : "$set-$node", "useIP" : true, "restart" : undefined, "pathOpts" : { "node" : 2, "set" : "testReplSet" } } ReplSetTest Starting.... Using Local IP for host: 127.0.0.4 Resetting db path '/data/db/testReplSet-2' Wed Jan 16 18:10:23.394 shell: started program /home/gregorv/Workspaces/10Gen/mongo/mongod --oplogSize 40 --port 31002 --noprealloc --smallfiles --rest --replSet testReplSet --dbpath /data/db/testReplSet-2 --nopreallocj --bind_ip 127.0.0.4 --setParameter enableTestCommands=1 m31002| note: noprealloc may hurt performance in many applications m31002| Wed Jan 16 18:10:23.418 [initandlisten] MongoDB starting : pid=5386 port=31002 dbpath=/data/db/testReplSet-2 64-bit host=nuwen m31002| Wed Jan 16 18:10:23.418 [initandlisten] m31002| Wed Jan 16 18:10:23.418 [initandlisten] ** NOTE: This is a development version (2.3.3-pre-) of MongoDB. m31002| Wed Jan 16 18:10:23.418 [initandlisten] ** Not recommended for production. m31002| Wed Jan 16 18:10:23.418 [initandlisten] m31002| Wed Jan 16 18:10:23.418 [initandlisten] db version v2.3.3-pre-, pdfile version 4.5 m31002| Wed Jan 16 18:10:23.418 [initandlisten] git version: b8c0b6ae71fb6c076101fa25ef915021ea26e156 m31002| Wed Jan 16 18:10:23.418 [initandlisten] build info: Linux nuwen 3.5.0-21-generic #32-Ubuntu SMP Tue Dec 11 18:51:59 UTC 2012 x86_64 BOOST_LIB_VERSION=1_49 m31002| Wed Jan 16 18:10:23.418 [initandlisten] allocator: tcmalloc m31002| Wed Jan 16 18:10:23.418 [initandlisten] options: { bind_ip: "127.0.0.4", dbpath: "/data/db/testReplSet-2", noprealloc: true, nopreallocj: true, oplogSize: 40, port: 31002, replSet: "testReplSet", rest: true, setParameter: [ "enableTestCommands=1" ], smallfiles: true } m31002| Wed Jan 16 18:10:23.491 [initandlisten] journal dir=/data/db/testReplSet-2/journal m31002| Wed Jan 16 18:10:23.491 [initandlisten] recover : no journal files present, no recovery needed m31002| Wed Jan 16 18:10:23.542 [FileAllocator] allocating new datafile /data/db/testReplSet-2/local.ns, filling with zeroes... m31002| Wed Jan 16 18:10:23.542 [FileAllocator] creating directory /data/db/testReplSet-2/_tmp m31002| Wed Jan 16 18:10:23.717 [FileAllocator] done allocating datafile /data/db/testReplSet-2/local.ns, size: 16MB, took 0.141 secs m31002| Wed Jan 16 18:10:23.717 [FileAllocator] allocating new datafile /data/db/testReplSet-2/local.0, filling with zeroes... m31002| Wed Jan 16 18:10:23.859 [FileAllocator] done allocating datafile /data/db/testReplSet-2/local.0, size: 16MB, took 0.141 secs m31002| Wed Jan 16 18:10:23.860 [initandlisten] command local.$cmd command: { create: "startup_log", size: 10485760, capped: true } ntoreturn:1 keyUpdates:0 reslen:37 317ms m31002| Wed Jan 16 18:10:23.860 [initandlisten] waiting for connections on port 31002 m31002| Wed Jan 16 18:10:23.860 [websvr] admin web console waiting for connections on port 32002 m31002| Wed Jan 16 18:10:23.862 [rsStart] replSet can't get local.system.replset config from self or any seed (EMPTYCONFIG) m31002| Wed Jan 16 18:10:23.862 [rsStart] replSet info you may need to run replSetInitiate -- rs.initiate() in the shell -- if that is not already done m31002| Wed Jan 16 18:10:23.995 [initandlisten] connection accepted from 127.0.0.1:57416 #1 (1 connection now open) [ connection to 127.0.0.2:31000, connection to 127.0.0.3:31001, connection to 127.0.0.4:31002 ] { "replSetInitiate" : { "_id" : "testReplSet", "members" : [ { "_id" : 0, "host" : "127.0.0.2:31000" }, { "_id" : 1, "host" : "127.0.0.3:31001" }, { "_id" : 2, "host" : "127.0.0.4:31002" } ] } } m31000| Wed Jan 16 18:10:23.999 [conn1] replSet replSetInitiate admin command received from client m31000| Wed Jan 16 18:10:23.999 [conn1] replSet replSetInitiate config object parses ok, 3 members specified m31001| Wed Jan 16 18:10:23.999 [initandlisten] connection accepted from 127.0.0.2:33222 #2 (2 connections now open) m31002| Wed Jan 16 18:10:24.000 [initandlisten] connection accepted from 127.0.0.2:33248 #2 (2 connections now open) m31000| Wed Jan 16 18:10:24.001 [conn1] replSet replSetInitiate all members seem up m31000| Wed Jan 16 18:10:24.001 [conn1] ****** m31000| Wed Jan 16 18:10:24.001 [conn1] creating replication oplog of size: 40MB... m31000| Wed Jan 16 18:10:24.001 [FileAllocator] allocating new datafile /data/db/testReplSet-0/local.1, filling with zeroes... m31000| Wed Jan 16 18:10:24.526 [FileAllocator] done allocating datafile /data/db/testReplSet-0/local.1, size: 64MB, took 0.524 secs m31000| Wed Jan 16 18:10:25.651 [conn1] ****** m31000| Wed Jan 16 18:10:25.651 [conn1] replSet info saving a newer config version to local.system.replset m31000| Wed Jan 16 18:10:25.652 [conn1] replSet saveConfigLocally done m31000| Wed Jan 16 18:10:25.652 [conn1] replSet replSetInitiate config now saved locally. Should come online in about a minute. m31000| Wed Jan 16 18:10:25.653 [conn1] build index local.replset.minvalid { _id: 1 } m31000| Wed Jan 16 18:10:25.653 [conn1] build index done. scanned 0 total records. 0 secs m31000| Wed Jan 16 18:10:25.653 [conn1] command admin.$cmd command: { replSetInitiate: { _id: "testReplSet", members: [ { _id: 0.0, host: "127.0.0.2:31000" }, { _id: 1.0, host: "127.0.0.3:31001" }, { _id: 2.0, host: "127.0.0.4:31002" } ] } } ntoreturn:1 keyUpdates:0 locks(micros) W:1652307 reslen:112 1654ms { "info" : "Config now saved locally. Should come online in about a minute.", "ok" : 1 } m31000| Wed Jan 16 18:10:32.374 [rsStart] replSet I am 127.0.0.2:31000 m31000| Wed Jan 16 18:10:32.375 [rsStart] replSet STARTUP2 m31000| Wed Jan 16 18:10:32.375 [rsHealthPoll] replSet member 127.0.0.4:31002 is up m31000| Wed Jan 16 18:10:32.375 [rsHealthPoll] replSet member 127.0.0.3:31001 is up m31001| Wed Jan 16 18:10:33.002 [rsStart] trying to contact 127.0.0.2:31000 m31000| Wed Jan 16 18:10:33.002 [initandlisten] connection accepted from 127.0.0.3:53932 #2 (2 connections now open) m31001| Wed Jan 16 18:10:33.003 [rsStart] replSet I am 127.0.0.3:31001 m31001| Wed Jan 16 18:10:33.003 [rsStart] replSet got config version 1 from a remote, saving locally m31001| Wed Jan 16 18:10:33.003 [rsStart] replSet info saving a newer config version to local.system.replset m31001| Wed Jan 16 18:10:33.229 [rsStart] replSet saveConfigLocally done m31001| Wed Jan 16 18:10:33.229 [rsStart] replSet STARTUP2 m31001| Wed Jan 16 18:10:33.230 [rsSync] ****** m31001| Wed Jan 16 18:10:33.230 [rsSync] creating replication oplog of size: 40MB... m31001| Wed Jan 16 18:10:33.271 [FileAllocator] allocating new datafile /data/db/testReplSet-1/local.1, filling with zeroes... m31000| Wed Jan 16 18:10:33.375 [rsSync] replSet SECONDARY m31001| Wed Jan 16 18:10:33.795 [FileAllocator] done allocating datafile /data/db/testReplSet-1/local.1, size: 64MB, took 0.523 secs m31002| Wed Jan 16 18:10:33.862 [rsStart] trying to contact 127.0.0.2:31000 m31000| Wed Jan 16 18:10:33.862 [initandlisten] connection accepted from 127.0.0.4:36050 #3 (3 connections now open) m31002| Wed Jan 16 18:10:33.863 [rsStart] replSet I am 127.0.0.4:31002 m31002| Wed Jan 16 18:10:33.863 [rsStart] replSet got config version 1 from a remote, saving locally m31002| Wed Jan 16 18:10:33.863 [rsStart] replSet info saving a newer config version to local.system.replset m31002| Wed Jan 16 18:10:34.237 [rsStart] replSet saveConfigLocally done m31002| Wed Jan 16 18:10:34.238 [rsStart] replSet STARTUP2 m31002| Wed Jan 16 18:10:34.238 [rsSync] ****** m31002| Wed Jan 16 18:10:34.238 [rsSync] creating replication oplog of size: 40MB... m31000| Wed Jan 16 18:10:34.375 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:10:34.375 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state STARTUP2 m31000| Wed Jan 16 18:10:34.375 [rsHealthPoll] replset info 127.0.0.4:31002 thinks that we are down m31000| Wed Jan 16 18:10:34.375 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state STARTUP2 m31000| Wed Jan 16 18:10:34.376 [rsMgr] not electing self, 127.0.0.4:31002 would veto with 'I don't think 127.0.0.2:31000 is electable' m31000| Wed Jan 16 18:10:34.376 [rsMgr] not electing self, 127.0.0.4:31002 would veto with 'I don't think 127.0.0.2:31000 is electable' m31002| Wed Jan 16 18:10:34.961 [FileAllocator] allocating new datafile /data/db/testReplSet-2/local.1, filling with zeroes... m31001| Wed Jan 16 18:10:34.978 [rsSync] ****** m31001| Wed Jan 16 18:10:34.978 [rsSync] replSet initial sync pending m31001| Wed Jan 16 18:10:34.978 [rsSync] replSet initial sync need a member to be primary or secondary to do our initial sync m31002| Wed Jan 16 18:10:35.004 [initandlisten] connection accepted from 127.0.0.3:34830 #3 (3 connections now open) m31001| Wed Jan 16 18:10:35.004 [rsHealthPoll] replSet member 127.0.0.2:31000 is up m31001| Wed Jan 16 18:10:35.004 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state SECONDARY m31001| Wed Jan 16 18:10:35.004 [rsHealthPoll] replset info 127.0.0.4:31002 thinks that we are down m31001| Wed Jan 16 18:10:35.004 [rsHealthPoll] replSet member 127.0.0.4:31002 is up m31001| Wed Jan 16 18:10:35.004 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state STARTUP2 m31002| Wed Jan 16 18:10:35.486 [FileAllocator] done allocating datafile /data/db/testReplSet-2/local.1, size: 64MB, took 0.524 secs m31002| Wed Jan 16 18:10:35.864 [rsHealthPoll] replSet member 127.0.0.2:31000 is up m31002| Wed Jan 16 18:10:35.864 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state SECONDARY m31001| Wed Jan 16 18:10:35.864 [initandlisten] connection accepted from 127.0.0.4:52034 #3 (3 connections now open) m31002| Wed Jan 16 18:10:35.864 [rsHealthPoll] replSet member 127.0.0.3:31001 is up m31002| Wed Jan 16 18:10:35.864 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state STARTUP2 m31002| Wed Jan 16 18:10:36.336 [rsSync] ****** m31002| Wed Jan 16 18:10:36.336 [rsSync] replSet initial sync pending m31002| Wed Jan 16 18:10:36.336 [rsSync] replSet initial sync need a member to be primary or secondary to do our initial sync m31000| Wed Jan 16 18:10:40.377 [rsMgr] replSet info electSelf 0 m31002| Wed Jan 16 18:10:40.377 [conn2] replSet RECOVERING m31002| Wed Jan 16 18:10:40.377 [conn2] replSet info voting yea for 127.0.0.2:31000 (0) m31001| Wed Jan 16 18:10:40.377 [conn2] replSet RECOVERING m31001| Wed Jan 16 18:10:40.377 [conn2] replSet info voting yea for 127.0.0.2:31000 (0) m31001| Wed Jan 16 18:10:41.005 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state RECOVERING m31000| Wed Jan 16 18:10:41.376 [rsMgr] replSet PRIMARY m31002| Wed Jan 16 18:10:41.865 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state PRIMARY m31002| Wed Jan 16 18:10:41.865 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state RECOVERING m31000| Wed Jan 16 18:10:42.377 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state RECOVERING m31000| Wed Jan 16 18:10:42.377 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state RECOVERING m31001| Wed Jan 16 18:10:43.005 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state PRIMARY m31001| Wed Jan 16 18:10:46.378 [conn2] end connection 127.0.0.2:33222 (2 connections now open) m31001| Wed Jan 16 18:10:46.378 [initandlisten] connection accepted from 127.0.0.2:35972 #4 (4 connections now open) m31000| Wed Jan 16 18:10:49.007 [conn2] end connection 127.0.0.3:53932 (2 connections now open) m31000| Wed Jan 16 18:10:49.007 [initandlisten] connection accepted from 127.0.0.3:55276 #4 (3 connections now open) m31000| Wed Jan 16 18:10:49.866 [conn3] end connection 127.0.0.4:36050 (2 connections now open) m31000| Wed Jan 16 18:10:49.867 [initandlisten] connection accepted from 127.0.0.4:54289 #5 (3 connections now open) m31001| Wed Jan 16 18:10:50.978 [rsSync] replSet initial sync pending m31001| Wed Jan 16 18:10:50.978 [rsSync] replSet syncing to: 127.0.0.2:31000 m31000| Wed Jan 16 18:10:50.979 [initandlisten] connection accepted from 127.0.0.3:38167 #6 (4 connections now open) m31001| Wed Jan 16 18:10:50.980 [rsSync] build index local.me { _id: 1 } m31001| Wed Jan 16 18:10:50.980 [rsSync] build index done. scanned 0 total records. 0 secs m31001| Wed Jan 16 18:10:50.981 [rsSync] replSet initial sync drop all databases m31001| Wed Jan 16 18:10:50.981 [rsSync] dropAllDatabasesExceptLocal 1 m31001| Wed Jan 16 18:10:50.981 [rsSync] replSet initial sync clone all databases m31001| Wed Jan 16 18:10:50.982 [rsSync] replSet initial sync data copy, starting syncup m31001| Wed Jan 16 18:10:50.982 [rsSync] oplog sync 1 of 3 m31001| Wed Jan 16 18:10:51.016 [rsSync] oplog sync 2 of 3 m31001| Wed Jan 16 18:10:51.016 [rsSync] replSet initial sync building indexes m31001| Wed Jan 16 18:10:51.016 [rsSync] oplog sync 3 of 3 m31001| Wed Jan 16 18:10:51.017 [rsSync] replSet initial sync finishing up m31001| Wed Jan 16 18:10:51.037 [rsSync] replSet set minValid=50f73361:1 m31001| Wed Jan 16 18:10:51.038 [rsSync] build index local.replset.minvalid { _id: 1 } m31001| Wed Jan 16 18:10:51.038 [rsSync] build index done. scanned 0 total records. 0 secs m31001| Wed Jan 16 18:10:51.039 [rsSync] replSet initial sync done m31000| Wed Jan 16 18:10:51.039 [conn6] end connection 127.0.0.3:38167 (3 connections now open) m31001| Wed Jan 16 18:10:51.231 [rsBackgroundSync] replSet syncing to: 127.0.0.2:31000 m31000| Wed Jan 16 18:10:51.232 [initandlisten] connection accepted from 127.0.0.3:46561 #7 (4 connections now open) m31001| Wed Jan 16 18:10:52.039 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.2:31000 m31000| Wed Jan 16 18:10:52.039 [initandlisten] connection accepted from 127.0.0.3:36592 #8 (5 connections now open) m31002| Wed Jan 16 18:10:52.336 [rsSync] replSet initial sync pending m31002| Wed Jan 16 18:10:52.336 [rsSync] replSet syncing to: 127.0.0.2:31000 m31000| Wed Jan 16 18:10:52.337 [initandlisten] connection accepted from 127.0.0.4:37075 #9 (6 connections now open) m31002| Wed Jan 16 18:10:52.338 [rsSync] build index local.me { _id: 1 } m31002| Wed Jan 16 18:10:52.339 [rsSync] build index done. scanned 0 total records. 0 secs m31002| Wed Jan 16 18:10:52.339 [rsSync] replSet initial sync drop all databases m31002| Wed Jan 16 18:10:52.339 [rsSync] dropAllDatabasesExceptLocal 1 m31002| Wed Jan 16 18:10:52.339 [rsSync] replSet initial sync clone all databases m31002| Wed Jan 16 18:10:52.340 [rsSync] replSet initial sync data copy, starting syncup m31002| Wed Jan 16 18:10:52.340 [rsSync] oplog sync 1 of 3 m31002| Wed Jan 16 18:10:52.340 [rsSync] oplog sync 2 of 3 m31002| Wed Jan 16 18:10:52.340 [rsSync] replSet initial sync building indexes m31002| Wed Jan 16 18:10:52.340 [rsSync] oplog sync 3 of 3 m31002| Wed Jan 16 18:10:52.341 [rsSync] replSet initial sync finishing up m31002| Wed Jan 16 18:10:52.363 [rsSync] replSet set minValid=50f73361:1 m31002| Wed Jan 16 18:10:52.363 [rsSync] build index local.replset.minvalid { _id: 1 } m31002| Wed Jan 16 18:10:52.364 [rsSync] build index done. scanned 0 total records. 0 secs m31002| Wed Jan 16 18:10:52.364 [rsSync] replSet initial sync done m31000| Wed Jan 16 18:10:52.364 [conn9] end connection 127.0.0.4:37075 (5 connections now open) m31001| Wed Jan 16 18:10:53.039 [rsSync] replSet SECONDARY m31000| Wed Jan 16 18:10:53.041 [slaveTracking] build index local.slaves { _id: 1 } m31000| Wed Jan 16 18:10:53.041 [slaveTracking] build index done. scanned 0 total records. 0 secs m31002| Wed Jan 16 18:10:53.239 [rsBackgroundSync] replSet syncing to: 127.0.0.2:31000 m31000| Wed Jan 16 18:10:53.239 [initandlisten] connection accepted from 127.0.0.4:52401 #10 (6 connections now open) m31002| Wed Jan 16 18:10:53.364 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.2:31000 m31000| Wed Jan 16 18:10:53.365 [initandlisten] connection accepted from 127.0.0.4:44650 #11 (7 connections now open) m31002| Wed Jan 16 18:10:53.868 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state SECONDARY m31002| Wed Jan 16 18:10:54.364 [rsSync] replSet SECONDARY m31000| Wed Jan 16 18:10:54.379 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state SECONDARY m31000| Wed Jan 16 18:10:54.379 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state SECONDARY ---- Started set! ---- Caught exception TypeError: Cannot read property '0' of undefined, this is normal after reconfig. Running: sudo bash -c dig +short undefined > /tmp/output Wed Jan 16 18:10:54.455 shell: started program sudo bash -c dig +short undefined > /tmp/output Running: sudo bash -c tc class add dev lo parent 1:1 classid 1:11 htb rate 1bps ceil 1000Mbps > /tmp/output Wed Jan 16 18:10:54.492 shell: started program sudo bash -c tc class add dev lo parent 1:1 classid 1:11 htb rate 1bps ceil 1000Mbps > /tmp/output Running: sudo bash -c tc filter add dev lo parent 1: prio 1 u32 match ip src 127.0.0.2 match ip dport 31002 0xFFFF flowid 1:11 > /tmp/output Wed Jan 16 18:10:54.507 shell: started program sudo bash -c tc filter add dev lo parent 1: prio 1 u32 match ip src 127.0.0.2 match ip dport 31002 0xFFFF flowid 1:11 > /tmp/output Running: sudo bash -c tc qdisc add dev lo parent 1:11 handle 11: netem > /tmp/output Wed Jan 16 18:10:54.520 shell: started program sudo bash -c tc qdisc add dev lo parent 1:11 handle 11: netem > /tmp/output Running: sudo bash -c tc qdisc change dev lo parent 1:11 handle 11: netem delay 500ms > /tmp/output Wed Jan 16 18:10:54.534 shell: started program sudo bash -c tc qdisc change dev lo parent 1:11 handle 11: netem delay 500ms > /tmp/output Running: sudo bash -c dig +short undefined > /tmp/output Wed Jan 16 18:10:54.550 shell: started program sudo bash -c dig +short undefined > /tmp/output Running: sudo bash -c tc class add dev lo parent 1:1 classid 1:12 htb rate 1bps ceil 1000Mbps > /tmp/output Wed Jan 16 18:10:54.581 shell: started program sudo bash -c tc class add dev lo parent 1:1 classid 1:12 htb rate 1bps ceil 1000Mbps > /tmp/output Running: sudo bash -c tc filter add dev lo parent 1: prio 1 u32 match ip src 127.0.0.3 match ip dport 31000 0xFFFF flowid 1:12 > /tmp/output Wed Jan 16 18:10:54.593 shell: started program sudo bash -c tc filter add dev lo parent 1: prio 1 u32 match ip src 127.0.0.3 match ip dport 31000 0xFFFF flowid 1:12 > /tmp/output Running: sudo bash -c tc qdisc add dev lo parent 1:12 handle 12: netem > /tmp/output Wed Jan 16 18:10:54.607 shell: started program sudo bash -c tc qdisc add dev lo parent 1:12 handle 12: netem > /tmp/output Running: sudo bash -c tc qdisc change dev lo parent 1:12 handle 12: netem delay 500ms > /tmp/output Wed Jan 16 18:10:54.621 shell: started program sudo bash -c tc qdisc change dev lo parent 1:12 handle 12: netem delay 500ms > /tmp/output Running: sudo bash -c dig +short undefined > /tmp/output Wed Jan 16 18:10:54.634 shell: started program sudo bash -c dig +short undefined > /tmp/output Running: sudo bash -c tc class add dev lo parent 1:1 classid 1:13 htb rate 1bps ceil 1000Mbps > /tmp/output Wed Jan 16 18:10:54.665 shell: started program sudo bash -c tc class add dev lo parent 1:1 classid 1:13 htb rate 1bps ceil 1000Mbps > /tmp/output Running: sudo bash -c tc filter add dev lo parent 1: prio 1 u32 match ip src 127.0.0.4 match ip dport 31001 0xFFFF flowid 1:13 > /tmp/output Wed Jan 16 18:10:54.677 shell: started program sudo bash -c tc filter add dev lo parent 1: prio 1 u32 match ip src 127.0.0.4 match ip dport 31001 0xFFFF flowid 1:13 > /tmp/output Running: sudo bash -c tc qdisc add dev lo parent 1:13 handle 13: netem > /tmp/output Wed Jan 16 18:10:54.687 shell: started program sudo bash -c tc qdisc add dev lo parent 1:13 handle 13: netem > /tmp/output Running: sudo bash -c tc qdisc change dev lo parent 1:13 handle 13: netem delay 500ms > /tmp/output Wed Jan 16 18:10:54.701 shell: started program sudo bash -c tc qdisc change dev lo parent 1:13 handle 13: netem delay 500ms > /tmp/output Running: sudo bash -c tc -s -d -iec qdisc show dev lo > /tmp/output Wed Jan 16 18:10:54.713 shell: started program sudo bash -c tc -s -d -iec qdisc show dev lo > /tmp/output qdisc htb 1: root refcnt 2 r2q 10 default 0 direct_packets_stat 1455 ver 3.17 Sent 251792 bytes 1455 pkt (dropped 0, overlimits 0 requeues 0) backlog 0b 0p requeues 0 qdisc netem 11: parent 1:11 limit 1000 delay 500.0ms Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) backlog 0b 0p requeues 0 qdisc netem 12: parent 1:12 limit 1000 delay 500.0ms Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) backlog 0b 0p requeues 0 qdisc netem 13: parent 1:13 limit 1000 delay 500.0ms Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) backlog 0b 0p requeues 0 m31001| Wed Jan 16 18:10:55.008 [rsHealthPoll] replSet member 127.0.0.4:31002 is now in state SECONDARY Running: sudo bash -c tc qdisc change dev lo parent 1:12 handle 12: netem delay 40000ms > /tmp/output Wed Jan 16 18:10:59.727 shell: started program sudo bash -c tc qdisc change dev lo parent 1:12 handle 12: netem delay 40000ms > /tmp/output m31002| Wed Jan 16 18:11:01.880 [conn2] end connection 127.0.0.2:33248 (2 connections now open) m31002| Wed Jan 16 18:11:02.380 [initandlisten] connection accepted from 127.0.0.2:51482 #4 (3 connections now open) m31002| Wed Jan 16 18:11:03.009 [conn3] end connection 127.0.0.3:34830 (2 connections now open) m31002| Wed Jan 16 18:11:03.010 [initandlisten] connection accepted from 127.0.0.3:53412 #5 (3 connections now open) m31000| Wed Jan 16 18:11:04.742 [FileAllocator] allocating new datafile /data/db/testReplSet-0/foo.ns, filling with zeroes... m31000| Wed Jan 16 18:11:04.891 [FileAllocator] done allocating datafile /data/db/testReplSet-0/foo.ns, size: 16MB, took 0.148 secs m31000| Wed Jan 16 18:11:04.891 [FileAllocator] allocating new datafile /data/db/testReplSet-0/foo.0, filling with zeroes... m31000| Wed Jan 16 18:11:05.032 [FileAllocator] done allocating datafile /data/db/testReplSet-0/foo.0, size: 16MB, took 0.141 secs m31000| Wed Jan 16 18:11:05.033 [conn1] build index foo.bar { _id: 1 } m31000| Wed Jan 16 18:11:05.033 [conn1] build index done. scanned 0 total records. 0 secs m31000| Wed Jan 16 18:11:05.034 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:291341 291ms m31002| Wed Jan 16 18:11:05.034 [FileAllocator] allocating new datafile /data/db/testReplSet-2/foo.ns, filling with zeroes... m31002| Wed Jan 16 18:11:05.208 [FileAllocator] done allocating datafile /data/db/testReplSet-2/foo.ns, size: 16MB, took 0.173 secs m31002| Wed Jan 16 18:11:05.208 [FileAllocator] allocating new datafile /data/db/testReplSet-2/foo.0, filling with zeroes... m31002| Wed Jan 16 18:11:05.358 [FileAllocator] done allocating datafile /data/db/testReplSet-2/foo.0, size: 16MB, took 0.15 secs m31002| Wed Jan 16 18:11:05.359 [repl writer worker 1] build index foo.bar { _id: 1 } m31002| Wed Jan 16 18:11:05.360 [repl writer worker 1] build index done. scanned 0 total records. 0 secs STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 44, "optime" : { "t" : 1358377865000, "i" : 10000 }, "optimeDate" : ISODate("2013-01-16T23:11:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 33, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 33, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 30, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:10:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:04Z"), "pingMs" : 500 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 43, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 30, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 30, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 30, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:05Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 42, "optime" : { "t" : 1358377865000, "i" : 10000 }, "optimeDate" : ISODate("2013-01-16T23:11:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:11:06.369 [conn3] end connection 127.0.0.4:52034 (2 connections now open) m31001| Wed Jan 16 18:11:06.870 [initandlisten] connection accepted from 127.0.0.4:55606 #5 (3 connections now open) m31000| Wed Jan 16 18:11:09.549 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:225148 112ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 48, "optime" : { "t" : 1358377869000, "i" : 8427 }, "optimeDate" : ISODate("2013-01-16T23:11:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 37, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 37, "optime" : { "t" : 1358377869000, "i" : 7705 }, "optimeDate" : ISODate("2013-01-16T23:11:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 34, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:10:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:08Z"), "pingMs" : 500 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 47, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 34, "optime" : { "t" : 1358377868000, "i" : 1567 }, "optimeDate" : ISODate("2013-01-16T23:11:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:09Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 34, "optime" : { "t" : 1358377869000, "i" : 7519 }, "optimeDate" : ISODate("2013-01-16T23:11:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:09Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 34, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 46, "optime" : { "t" : 1358377869000, "i" : 8427 }, "optimeDate" : ISODate("2013-01-16T23:11:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:11:10.007 [rsHealthPoll] DBClientCursor::init call() failed m31001| Wed Jan 16 18:11:10.007 [rsHealthPoll] replset info 127.0.0.2:31000 heartbeat failed, retrying m31001| Wed Jan 16 18:11:11.007 [rsHealthPoll] replSet info 127.0.0.2:31000 is down (or slow to respond): m31001| Wed Jan 16 18:11:11.007 [rsHealthPoll] replSet member 127.0.0.2:31000 is now in state DOWN m31000| Wed Jan 16 18:11:12.383 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 53, "optime" : { "t" : 1358377874000, "i" : 602 }, "optimeDate" : ISODate("2013-01-16T23:11:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 42, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 42, "optime" : { "t" : 1358377869000, "i" : 8427 }, "optimeDate" : ISODate("2013-01-16T23:11:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:13Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 52, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 39, "optime" : { "t" : 1358377872000, "i" : 430 }, "optimeDate" : ISODate("2013-01-16T23:11:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 39, "optime" : { "t" : 1358377873000, "i" : 7714 }, "optimeDate" : ISODate("2013-01-16T23:11:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 39, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:13Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 51, "optime" : { "t" : 1358377874000, "i" : 602 }, "optimeDate" : ISODate("2013-01-16T23:11:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:11:14.383 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:11:16.383 [conn4] end connection 127.0.0.2:35972 (2 connections now open) m31001| Wed Jan 16 18:11:16.384 [initandlisten] connection accepted from 127.0.0.2:45803 #6 (3 connections now open) m31000| Wed Jan 16 18:11:16.384 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 57, "optime" : { "t" : 1358377878000, "i" : 2374 }, "optimeDate" : ISODate("2013-01-16T23:11:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 46, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 46, "optime" : { "t" : 1358377877000, "i" : 1987 }, "optimeDate" : ISODate("2013-01-16T23:11:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:17Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 56, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 43, "optime" : { "t" : 1358377874000, "i" : 602 }, "optimeDate" : ISODate("2013-01-16T23:11:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 43, "optime" : { "t" : 1358377877000, "i" : 6759 }, "optimeDate" : ISODate("2013-01-16T23:11:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 43, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 55, "optime" : { "t" : 1358377878000, "i" : 2162 }, "optimeDate" : ISODate("2013-01-16T23:11:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:11:18.384 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:11:19.872 [conn5] end connection 127.0.0.4:54289 (6 connections now open) m31000| Wed Jan 16 18:11:19.872 [initandlisten] connection accepted from 127.0.0.4:55363 #12 (7 connections now open) m31000| Wed Jan 16 18:11:20.385 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:11:22.150 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:417380 208ms STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:11:22.385 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:11:22.502 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:496093 248ms { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 61, "optime" : { "t" : 1358377882000, "i" : 4313 }, "optimeDate" : ISODate("2013-01-16T23:11:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 50, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 50, "optime" : { "t" : 1358377882000, "i" : 2407 }, "optimeDate" : ISODate("2013-01-16T23:11:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 60, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 47, "optime" : { "t" : 1358377878000, "i" : 2374 }, "optimeDate" : ISODate("2013-01-16T23:11:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:21Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 47, "optime" : { "t" : 1358377881000, "i" : 5195 }, "optimeDate" : ISODate("2013-01-16T23:11:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 47, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 59, "optime" : { "t" : 1358377882000, "i" : 4313 }, "optimeDate" : ISODate("2013-01-16T23:11:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:11:23.007 [rsHealthPoll] replset info 127.0.0.2:31000 heartbeat failed, retrying m31000| Wed Jan 16 18:11:24.386 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:11:26.386 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 65, "optime" : { "t" : 1358377886000, "i" : 6817 }, "optimeDate" : ISODate("2013-01-16T23:11:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 54, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 54, "optime" : { "t" : 1358377882000, "i" : 4313 }, "optimeDate" : ISODate("2013-01-16T23:11:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:25Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 64, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 51, "optime" : { "t" : 1358377882000, "i" : 4313 }, "optimeDate" : ISODate("2013-01-16T23:11:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 51, "optime" : { "t" : 1358377885000, "i" : 2308 }, "optimeDate" : ISODate("2013-01-16T23:11:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 51, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 63, "optime" : { "t" : 1358377886000, "i" : 6817 }, "optimeDate" : ISODate("2013-01-16T23:11:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:11:28.386 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:11:30.387 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 69, "optime" : { "t" : 1358377890000, "i" : 7874 }, "optimeDate" : ISODate("2013-01-16T23:11:30Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 58, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 58, "optime" : { "t" : 1358377889000, "i" : 1088 }, "optimeDate" : ISODate("2013-01-16T23:11:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 68, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 55, "optime" : { "t" : 1358377886000, "i" : 6817 }, "optimeDate" : ISODate("2013-01-16T23:11:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:29Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 55, "optime" : { "t" : 1358377889000, "i" : 1088 }, "optimeDate" : ISODate("2013-01-16T23:11:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:29Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 55, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 67, "optime" : { "t" : 1358377890000, "i" : 7661 }, "optimeDate" : ISODate("2013-01-16T23:11:30Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:11:31.755 [rsBackgroundSync] Socket recv() timeout 127.0.0.2:31000 m31001| Wed Jan 16 18:11:31.755 [rsBackgroundSync] SocketException: remote: 127.0.0.2:31000 error: 9001 socket exception [3] server [127.0.0.2:31000] m31001| Wed Jan 16 18:11:31.755 [rsBackgroundSync] replSet db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000 m31000| Wed Jan 16 18:11:32.387 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:11:33.015 [conn5] end connection 127.0.0.3:53412 (2 connections now open) m31002| Wed Jan 16 18:11:33.015 [initandlisten] connection accepted from 127.0.0.3:44026 #6 (3 connections now open) m31000| Wed Jan 16 18:11:34.387 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 74, "optime" : { "t" : 1358377895000, "i" : 1219 }, "optimeDate" : ISODate("2013-01-16T23:11:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 63, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 63, "optime" : { "t" : 1358377894000, "i" : 7690 }, "optimeDate" : ISODate("2013-01-16T23:11:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 73, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "errmsg" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 60, "optime" : { "t" : 1358377895000, "i" : 250 }, "optimeDate" : ISODate("2013-01-16T23:11:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 60, "optime" : { "t" : 1358377890000, "i" : 7874 }, "optimeDate" : ISODate("2013-01-16T23:11:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 60, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:35Z"), "pingMs" : 500, "lastHeartbeatMessage" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 72, "optime" : { "t" : 1358377895000, "i" : 1219 }, "optimeDate" : ISODate("2013-01-16T23:11:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:11:36.007 [rsHealthPoll] replset info 127.0.0.2:31000 heartbeat failed, retrying m31000| Wed Jan 16 18:11:36.388 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:11:38.388 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 78, "optime" : { "t" : 1358377899000, "i" : 2465 }, "optimeDate" : ISODate("2013-01-16T23:11:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 67, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 67, "optime" : { "t" : 1358377895000, "i" : 1219 }, "optimeDate" : ISODate("2013-01-16T23:11:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:37Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 77, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "errmsg" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 64, "optime" : { "t" : 1358377898000, "i" : 7296 }, "optimeDate" : ISODate("2013-01-16T23:11:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 64, "optime" : { "t" : 1358377895000, "i" : 1219 }, "optimeDate" : ISODate("2013-01-16T23:11:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 64, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:39Z"), "pingMs" : 500, "lastHeartbeatMessage" : "db exception in producer: 10278 dbclient error communicating with server: 127.0.0.2:31000" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 76, "optime" : { "t" : 1358377899000, "i" : 2383 }, "optimeDate" : ISODate("2013-01-16T23:11:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.3:31001", "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:11:39.886 [conn4] end connection 127.0.0.2:51482 (2 connections now open) m31002| Wed Jan 16 18:11:40.386 [initandlisten] connection accepted from 127.0.0.2:55070 #7 (3 connections now open) m31000| Wed Jan 16 18:11:40.388 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:11:41.755 [rsBackgroundSync] replSet syncing to: 127.0.0.4:31002 m31002| Wed Jan 16 18:11:41.755 [initandlisten] connection accepted from 127.0.0.3:39284 #8 (4 connections now open) m31001| Wed Jan 16 18:11:41.757 [FileAllocator] allocating new datafile /data/db/testReplSet-1/foo.ns, filling with zeroes... m31001| Wed Jan 16 18:11:41.911 [FileAllocator] done allocating datafile /data/db/testReplSet-1/foo.ns, size: 16MB, took 0.153 secs m31001| Wed Jan 16 18:11:41.911 [FileAllocator] allocating new datafile /data/db/testReplSet-1/foo.0, filling with zeroes... m31001| Wed Jan 16 18:11:42.052 [FileAllocator] done allocating datafile /data/db/testReplSet-1/foo.0, size: 16MB, took 0.141 secs m31001| Wed Jan 16 18:11:42.054 [repl writer worker 1] build index foo.bar { _id: 1 } m31001| Wed Jan 16 18:11:42.054 [repl writer worker 1] build index done. scanned 0 total records. 0 secs m31001| Wed Jan 16 18:11:42.086 [rsSyncNotifier] replset setting oplog notifier to 127.0.0.4:31002 m31002| Wed Jan 16 18:11:42.087 [initandlisten] connection accepted from 127.0.0.3:51250 #9 (5 connections now open) m31000| Wed Jan 16 18:11:42.187 [initandlisten] connection accepted from 127.0.0.4:40952 #13 (8 connections now open) m31002| Wed Jan 16 18:11:42.187 [rsGhostSync] handshake between 1 and 127.0.0.2:31000 m31000| Wed Jan 16 18:11:42.389 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:11:42.897 [FileAllocator] allocating new datafile /data/db/testReplSet-0/foo.1, filling with zeroes... m31000| Wed Jan 16 18:11:43.180 [FileAllocator] done allocating datafile /data/db/testReplSet-0/foo.1, size: 32MB, took 0.282 secs m31000| Wed Jan 16 18:11:43.180 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:283225 283ms m31002| Wed Jan 16 18:11:43.181 [FileAllocator] allocating new datafile /data/db/testReplSet-2/foo.1, filling with zeroes... STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 82, "optime" : { "t" : 1358377903000, "i" : 4844 }, "optimeDate" : ISODate("2013-01-16T23:11:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 71, "optime" : { "t" : 1358377865000, "i" : 5003 }, "optimeDate" : ISODate("2013-01-16T23:11:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 71, "optime" : { "t" : 1358377902000, "i" : 5092 }, "optimeDate" : ISODate("2013-01-16T23:11:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:43Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 81, "optime" : { "t" : 1358377881000, "i" : 9 }, "optimeDate" : ISODate("2013-01-16T23:11:21Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 68, "optime" : { "t" : 1358377902000, "i" : 5156 }, "optimeDate" : ISODate("2013-01-16T23:11:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 68, "optime" : { "t" : 1358377899000, "i" : 2465 }, "optimeDate" : ISODate("2013-01-16T23:11:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 68, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:43Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 80, "optime" : { "t" : 1358377902000, "i" : 5156 }, "optimeDate" : ISODate("2013-01-16T23:11:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:11:43.464 [FileAllocator] done allocating datafile /data/db/testReplSet-2/foo.1, size: 32MB, took 0.283 secs m31002| Wed Jan 16 18:11:43.465 [slaveTracking] build index local.slaves { _id: 1 } m31002| Wed Jan 16 18:11:43.465 [slaveTracking] build index done. scanned 0 total records. 0 secs m31001| Wed Jan 16 18:11:44.375 [conn5] end connection 127.0.0.4:55606 (2 connections now open) m31000| Wed Jan 16 18:11:44.389 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:11:44.875 [initandlisten] connection accepted from 127.0.0.4:48006 #7 (3 connections now open) m31001| Wed Jan 16 18:11:45.226 [FileAllocator] allocating new datafile /data/db/testReplSet-1/foo.1, filling with zeroes... m31001| Wed Jan 16 18:11:45.492 [FileAllocator] done allocating datafile /data/db/testReplSet-1/foo.1, size: 32MB, took 0.265 secs m31001| Wed Jan 16 18:11:46.389 [conn6] end connection 127.0.0.2:45803 (2 connections now open) m31001| Wed Jan 16 18:11:46.389 [initandlisten] connection accepted from 127.0.0.2:34899 #8 (3 connections now open) m31000| Wed Jan 16 18:11:46.390 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:11:47.018 [rsMgr] not electing self, we are not freshest STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 86, "optime" : { "t" : 1358377907000, "i" : 5782 }, "optimeDate" : ISODate("2013-01-16T23:11:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 75, "optime" : { "t" : 1358377903000, "i" : 4844 }, "optimeDate" : ISODate("2013-01-16T23:11:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 75, "optime" : { "t" : 1358377903000, "i" : 4844 }, "optimeDate" : ISODate("2013-01-16T23:11:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:45Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:47Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 85, "optime" : { "t" : 1358377906000, "i" : 3112 }, "optimeDate" : ISODate("2013-01-16T23:11:46Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 72, "optime" : { "t" : 1358377906000, "i" : 4209 }, "optimeDate" : ISODate("2013-01-16T23:11:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:47Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 72, "optime" : { "t" : 1358377903000, "i" : 4844 }, "optimeDate" : ISODate("2013-01-16T23:11:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 72, "optime" : { "t" : 1358377906000, "i" : 3112 }, "optimeDate" : ISODate("2013-01-16T23:11:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 84, "optime" : { "t" : 1358377907000, "i" : 5782 }, "optimeDate" : ISODate("2013-01-16T23:11:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:11:48.390 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:11:49.007 [rsHealthPoll] replset info 127.0.0.2:31000 heartbeat failed, retrying m31000| Wed Jan 16 18:11:49.878 [conn12] end connection 127.0.0.4:55363 (7 connections now open) m31000| Wed Jan 16 18:11:49.878 [initandlisten] connection accepted from 127.0.0.4:46337 #14 (8 connections now open) m31000| Wed Jan 16 18:11:50.007 [conn4] end connection 127.0.0.3:55276 (7 connections now open) m31000| Wed Jan 16 18:11:50.390 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 90, "optime" : { "t" : 1358377911000, "i" : 6648 }, "optimeDate" : ISODate("2013-01-16T23:11:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 79, "optime" : { "t" : 1358377907000, "i" : 5782 }, "optimeDate" : ISODate("2013-01-16T23:11:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 79, "optime" : { "t" : 1358377907000, "i" : 5782 }, "optimeDate" : ISODate("2013-01-16T23:11:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:51Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 89, "optime" : { "t" : 1358377910000, "i" : 2350 }, "optimeDate" : ISODate("2013-01-16T23:11:50Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 76, "optime" : { "t" : 1358377910000, "i" : 2458 }, "optimeDate" : ISODate("2013-01-16T23:11:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 76, "optime" : { "t" : 1358377907000, "i" : 5782 }, "optimeDate" : ISODate("2013-01-16T23:11:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 76, "optime" : { "t" : 1358377907000, "i" : 5782 }, "optimeDate" : ISODate("2013-01-16T23:11:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:51Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 88, "optime" : { "t" : 1358377911000, "i" : 6648 }, "optimeDate" : ISODate("2013-01-16T23:11:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:11:52.391 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:11:53.019 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31000| Wed Jan 16 18:11:54.391 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 94, "optime" : { "t" : 1358377915000, "i" : 8179 }, "optimeDate" : ISODate("2013-01-16T23:11:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 83, "optime" : { "t" : 1358377911000, "i" : 6648 }, "optimeDate" : ISODate("2013-01-16T23:11:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 83, "optime" : { "t" : 1358377915000, "i" : 3032 }, "optimeDate" : ISODate("2013-01-16T23:11:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:55Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:55Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 93, "optime" : { "t" : 1358377914000, "i" : 473 }, "optimeDate" : ISODate("2013-01-16T23:11:54Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 80, "optime" : { "t" : 1358377915000, "i" : 107 }, "optimeDate" : ISODate("2013-01-16T23:11:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:11:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 80, "optime" : { "t" : 1358377915000, "i" : 7309 }, "optimeDate" : ISODate("2013-01-16T23:11:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 80, "optime" : { "t" : 1358377911000, "i" : 6648 }, "optimeDate" : ISODate("2013-01-16T23:11:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:55Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 92, "optime" : { "t" : 1358377915000, "i" : 7311 }, "optimeDate" : ISODate("2013-01-16T23:11:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:11:56.392 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:11:58.392 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:11:59.020 [rsMgr] not electing self, we are not freshest STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 99, "optime" : { "t" : 1358377920000, "i" : 195 }, "optimeDate" : ISODate("2013-01-16T23:12:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 88, "optime" : { "t" : 1358377915000, "i" : 8179 }, "optimeDate" : ISODate("2013-01-16T23:11:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 88, "optime" : { "t" : 1358377915000, "i" : 8179 }, "optimeDate" : ISODate("2013-01-16T23:11:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:59Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:00Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 98, "optime" : { "t" : 1358377918000, "i" : 67 }, "optimeDate" : ISODate("2013-01-16T23:11:58Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 85, "optime" : { "t" : 1358377918000, "i" : 197 }, "optimeDate" : ISODate("2013-01-16T23:11:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:11:59Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 85, "optime" : { "t" : 1358377919000, "i" : 7681 }, "optimeDate" : ISODate("2013-01-16T23:11:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 85, "optime" : { "t" : 1358377918000, "i" : 67 }, "optimeDate" : ISODate("2013-01-16T23:11:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:11:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 97, "optime" : { "t" : 1358377920000, "i" : 120 }, "optimeDate" : ISODate("2013-01-16T23:12:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:12:00.392 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:12:02.007 [rsHealthPoll] replset info 127.0.0.2:31000 heartbeat failed, retrying m31000| Wed Jan 16 18:12:02.393 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:12:03.020 [conn6] end connection 127.0.0.3:44026 (4 connections now open) m31002| Wed Jan 16 18:12:03.020 [initandlisten] connection accepted from 127.0.0.3:40125 #10 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 103, "optime" : { "t" : 1358377924000, "i" : 2335 }, "optimeDate" : ISODate("2013-01-16T23:12:04Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 92, "optime" : { "t" : 1358377920000, "i" : 195 }, "optimeDate" : ISODate("2013-01-16T23:12:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 92, "optime" : { "t" : 1358377920000, "i" : 195 }, "optimeDate" : ISODate("2013-01-16T23:12:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:03Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:04Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 102, "optime" : { "t" : 1358377923000, "i" : 7526 }, "optimeDate" : ISODate("2013-01-16T23:12:03Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 89, "optime" : { "t" : 1358377920000, "i" : 195 }, "optimeDate" : ISODate("2013-01-16T23:12:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 89, "optime" : { "t" : 1358377923000, "i" : 6625 }, "optimeDate" : ISODate("2013-01-16T23:12:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 89, "optime" : { "t" : 1358377920000, "i" : 195 }, "optimeDate" : ISODate("2013-01-16T23:12:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:03Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 101, "optime" : { "t" : 1358377924000, "i" : 2330 }, "optimeDate" : ISODate("2013-01-16T23:12:04Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:12:04.393 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:12:05.021 [rsMgr] not electing self, we are not freshest m31000| Wed Jan 16 18:12:06.393 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:12:08.394 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 107, "optime" : { "t" : 1358377928000, "i" : 3847 }, "optimeDate" : ISODate("2013-01-16T23:12:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 96, "optime" : { "t" : 1358377927000, "i" : 6038 }, "optimeDate" : ISODate("2013-01-16T23:12:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 96, "optime" : { "t" : 1358377927000, "i" : 4897 }, "optimeDate" : ISODate("2013-01-16T23:12:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:08Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 106, "optime" : { "t" : 1358377927000, "i" : 6038 }, "optimeDate" : ISODate("2013-01-16T23:12:07Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 93, "optime" : { "t" : 1358377924000, "i" : 2335 }, "optimeDate" : ISODate("2013-01-16T23:12:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:07Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 93, "optime" : { "t" : 1358377927000, "i" : 4897 }, "optimeDate" : ISODate("2013-01-16T23:12:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:07Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 93, "optime" : { "t" : 1358377924000, "i" : 2335 }, "optimeDate" : ISODate("2013-01-16T23:12:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 105, "optime" : { "t" : 1358377928000, "i" : 3847 }, "optimeDate" : ISODate("2013-01-16T23:12:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:12:10.394 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:12:11.022 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31000| Wed Jan 16 18:12:12.394 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 111, "optime" : { "t" : 1358377932000, "i" : 5916 }, "optimeDate" : ISODate("2013-01-16T23:12:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 100, "optime" : { "t" : 1358377932000, "i" : 142 }, "optimeDate" : ISODate("2013-01-16T23:12:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 100, "optime" : { "t" : 1358377928000, "i" : 3847 }, "optimeDate" : ISODate("2013-01-16T23:12:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:11Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:12Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 110, "optime" : { "t" : 1358377932000, "i" : 142 }, "optimeDate" : ISODate("2013-01-16T23:12:12Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 97, "optime" : { "t" : 1358377928000, "i" : 3847 }, "optimeDate" : ISODate("2013-01-16T23:12:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:12Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 97, "optime" : { "t" : 1358377931000, "i" : 3011 }, "optimeDate" : ISODate("2013-01-16T23:12:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 97, "optime" : { "t" : 1358377932000, "i" : 142 }, "optimeDate" : ISODate("2013-01-16T23:12:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 109, "optime" : { "t" : 1358377932000, "i" : 5916 }, "optimeDate" : ISODate("2013-01-16T23:12:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:12:14.395 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:12:15.007 [rsHealthPoll] replset info 127.0.0.2:31000 heartbeat failed, retrying m31001| Wed Jan 16 18:12:16.007 [rsMgr] not electing self, we are not freshest m31001| Wed Jan 16 18:12:16.395 [conn8] end connection 127.0.0.2:34899 (2 connections now open) m31001| Wed Jan 16 18:12:16.395 [initandlisten] connection accepted from 127.0.0.2:59501 #9 (3 connections now open) m31000| Wed Jan 16 18:12:16.395 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 115, "optime" : { "t" : 1358377936000, "i" : 7234 }, "optimeDate" : ISODate("2013-01-16T23:12:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 104, "optime" : { "t" : 1358377936000, "i" : 338 }, "optimeDate" : ISODate("2013-01-16T23:12:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 104, "optime" : { "t" : 1358377932000, "i" : 5916 }, "optimeDate" : ISODate("2013-01-16T23:12:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:15Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:16Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 114, "optime" : { "t" : 1358377936000, "i" : 338 }, "optimeDate" : ISODate("2013-01-16T23:12:16Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 101, "optime" : { "t" : 1358377932000, "i" : 5916 }, "optimeDate" : ISODate("2013-01-16T23:12:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 101, "optime" : { "t" : 1358377935000, "i" : 1662 }, "optimeDate" : ISODate("2013-01-16T23:12:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 101, "optime" : { "t" : 1358377932000, "i" : 5916 }, "optimeDate" : ISODate("2013-01-16T23:12:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:15Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 113, "optime" : { "t" : 1358377936000, "i" : 6968 }, "optimeDate" : ISODate("2013-01-16T23:12:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:12:17.891 [conn7] end connection 127.0.0.2:55070 (4 connections now open) m31002| Wed Jan 16 18:12:18.391 [initandlisten] connection accepted from 127.0.0.2:33901 #11 (5 connections now open) m31000| Wed Jan 16 18:12:18.396 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:12:19.883 [conn14] end connection 127.0.0.4:46337 (6 connections now open) m31000| Wed Jan 16 18:12:19.883 [initandlisten] connection accepted from 127.0.0.4:51448 #15 (7 connections now open) m31000| Wed Jan 16 18:12:20.353 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:895526 447ms m31000| Wed Jan 16 18:12:20.396 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 119, "optime" : { "t" : 1358377940000, "i" : 9713 }, "optimeDate" : ISODate("2013-01-16T23:12:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 108, "optime" : { "t" : 1358377939000, "i" : 286 }, "optimeDate" : ISODate("2013-01-16T23:12:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 108, "optime" : { "t" : 1358377936000, "i" : 7234 }, "optimeDate" : ISODate("2013-01-16T23:12:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:19Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:20Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 118, "optime" : { "t" : 1358377939000, "i" : 286 }, "optimeDate" : ISODate("2013-01-16T23:12:19Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 105, "optime" : { "t" : 1358377936000, "i" : 7234 }, "optimeDate" : ISODate("2013-01-16T23:12:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:19Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 105, "optime" : { "t" : 1358377939000, "i" : 127 }, "optimeDate" : ISODate("2013-01-16T23:12:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 105, "optime" : { "t" : 1358377936000, "i" : 7234 }, "optimeDate" : ISODate("2013-01-16T23:12:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 117, "optime" : { "t" : 1358377940000, "i" : 9713 }, "optimeDate" : ISODate("2013-01-16T23:12:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:12:21.024 [rsMgr] not electing self, we are not freshest m31000| Wed Jan 16 18:12:22.087 [conn8] end connection 127.0.0.3:36592 (6 connections now open) m31001| Wed Jan 16 18:12:22.380 [conn7] end connection 127.0.0.4:48006 (2 connections now open) m31000| Wed Jan 16 18:12:22.396 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:12:22.880 [initandlisten] connection accepted from 127.0.0.4:58713 #10 (3 connections now open) m31000| Wed Jan 16 18:12:24.397 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 124, "optime" : { "t" : 1358377945000, "i" : 564 }, "optimeDate" : ISODate("2013-01-16T23:12:25Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 113, "optime" : { "t" : 1358377944000, "i" : 767 }, "optimeDate" : ISODate("2013-01-16T23:12:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 113, "optime" : { "t" : 1358377940000, "i" : 9713 }, "optimeDate" : ISODate("2013-01-16T23:12:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:23Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:25Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 123, "optime" : { "t" : 1358377944000, "i" : 767 }, "optimeDate" : ISODate("2013-01-16T23:12:24Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 110, "optime" : { "t" : 1358377945000, "i" : 142 }, "optimeDate" : ISODate("2013-01-16T23:12:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:25Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 110, "optime" : { "t" : 1358377943000, "i" : 169 }, "optimeDate" : ISODate("2013-01-16T23:12:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 110, "optime" : { "t" : 1358377940000, "i" : 9713 }, "optimeDate" : ISODate("2013-01-16T23:12:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:25Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 122, "optime" : { "t" : 1358377945000, "i" : 407 }, "optimeDate" : ISODate("2013-01-16T23:12:25Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:12:26.397 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:12:27.025 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31001| Wed Jan 16 18:12:28.007 [rsHealthPoll] replset info 127.0.0.2:31000 heartbeat failed, retrying m31000| Wed Jan 16 18:12:28.397 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 128, "optime" : { "t" : 1358377949000, "i" : 1769 }, "optimeDate" : ISODate("2013-01-16T23:12:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 117, "optime" : { "t" : 1358377948000, "i" : 158 }, "optimeDate" : ISODate("2013-01-16T23:12:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 117, "optime" : { "t" : 1358377948000, "i" : 2931 }, "optimeDate" : ISODate("2013-01-16T23:12:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:29Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 127, "optime" : { "t" : 1358377948000, "i" : 158 }, "optimeDate" : ISODate("2013-01-16T23:12:28Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 114, "optime" : { "t" : 1358377949000, "i" : 154 }, "optimeDate" : ISODate("2013-01-16T23:12:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 114, "optime" : { "t" : 1358377945000, "i" : 564 }, "optimeDate" : ISODate("2013-01-16T23:12:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 114, "optime" : { "t" : 1358377945000, "i" : 564 }, "optimeDate" : ISODate("2013-01-16T23:12:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:29Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 126, "optime" : { "t" : 1358377949000, "i" : 1615 }, "optimeDate" : ISODate("2013-01-16T23:12:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:12:30.398 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:12:32.398 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:12:33.025 [conn10] end connection 127.0.0.3:40125 (4 connections now open) m31002| Wed Jan 16 18:12:33.026 [initandlisten] connection accepted from 127.0.0.3:49663 #12 (5 connections now open) m31001| Wed Jan 16 18:12:33.026 [rsMgr] not electing self, we are not freshest STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 132, "optime" : { "t" : 1358377953000, "i" : 3534 }, "optimeDate" : ISODate("2013-01-16T23:12:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 121, "optime" : { "t" : 1358377949000, "i" : 1769 }, "optimeDate" : ISODate("2013-01-16T23:12:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 121, "optime" : { "t" : 1358377953000, "i" : 3473 }, "optimeDate" : ISODate("2013-01-16T23:12:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:33Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 131, "optime" : { "t" : 1358377953000, "i" : 1567 }, "optimeDate" : ISODate("2013-01-16T23:12:33Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 118, "optime" : { "t" : 1358377953000, "i" : 350 }, "optimeDate" : ISODate("2013-01-16T23:12:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 118, "optime" : { "t" : 1358377949000, "i" : 1769 }, "optimeDate" : ISODate("2013-01-16T23:12:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:33Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 118, "optime" : { "t" : 1358377949000, "i" : 1769 }, "optimeDate" : ISODate("2013-01-16T23:12:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:33Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 130, "optime" : { "t" : 1358377953000, "i" : 3502 }, "optimeDate" : ISODate("2013-01-16T23:12:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:12:34.398 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:12:36.399 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 136, "optime" : { "t" : 1358377957000, "i" : 5330 }, "optimeDate" : ISODate("2013-01-16T23:12:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 125, "optime" : { "t" : 1358377953000, "i" : 3534 }, "optimeDate" : ISODate("2013-01-16T23:12:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 125, "optime" : { "t" : 1358377953000, "i" : 3534 }, "optimeDate" : ISODate("2013-01-16T23:12:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:37Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 135, "optime" : { "t" : 1358377957000, "i" : 1680 }, "optimeDate" : ISODate("2013-01-16T23:12:37Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 122, "optime" : { "t" : 1358377956000, "i" : 4194 }, "optimeDate" : ISODate("2013-01-16T23:12:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 122, "optime" : { "t" : 1358377953000, "i" : 3534 }, "optimeDate" : ISODate("2013-01-16T23:12:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:35Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 122, "optime" : { "t" : 1358377953000, "i" : 3534 }, "optimeDate" : ISODate("2013-01-16T23:12:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:37Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 134, "optime" : { "t" : 1358377957000, "i" : 5330 }, "optimeDate" : ISODate("2013-01-16T23:12:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:12:38.399 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:12:39.027 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31000| Wed Jan 16 18:12:40.400 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:12:41.007 [rsHealthPoll] replset info 127.0.0.2:31000 heartbeat failed, retrying STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 140, "optime" : { "t" : 1358377961000, "i" : 6332 }, "optimeDate" : ISODate("2013-01-16T23:12:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 129, "optime" : { "t" : 1358377957000, "i" : 5330 }, "optimeDate" : ISODate("2013-01-16T23:12:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 129, "optime" : { "t" : 1358377960000, "i" : 2594 }, "optimeDate" : ISODate("2013-01-16T23:12:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:41Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 139, "optime" : { "t" : 1358377961000, "i" : 2067 }, "optimeDate" : ISODate("2013-01-16T23:12:41Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 126, "optime" : { "t" : 1358377961000, "i" : 231 }, "optimeDate" : ISODate("2013-01-16T23:12:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:41Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 126, "optime" : { "t" : 1358377957000, "i" : 5330 }, "optimeDate" : ISODate("2013-01-16T23:12:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 126, "optime" : { "t" : 1358377957000, "i" : 5330 }, "optimeDate" : ISODate("2013-01-16T23:12:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:41Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 138, "optime" : { "t" : 1358377961000, "i" : 6332 }, "optimeDate" : ISODate("2013-01-16T23:12:41Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:12:42.400 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:12:44.400 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:12:45.028 [rsMgr] not electing self, we are not freshest STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 144, "optime" : { "t" : 1358377965000, "i" : 7780 }, "optimeDate" : ISODate("2013-01-16T23:12:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 133, "optime" : { "t" : 1358377961000, "i" : 6332 }, "optimeDate" : ISODate("2013-01-16T23:12:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 133, "optime" : { "t" : 1358377961000, "i" : 6332 }, "optimeDate" : ISODate("2013-01-16T23:12:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:43Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:45Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 143, "optime" : { "t" : 1358377965000, "i" : 2697 }, "optimeDate" : ISODate("2013-01-16T23:12:45Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 130, "optime" : { "t" : 1358377965000, "i" : 471 }, "optimeDate" : ISODate("2013-01-16T23:12:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:45Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 130, "optime" : { "t" : 1358377961000, "i" : 6332 }, "optimeDate" : ISODate("2013-01-16T23:12:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 130, "optime" : { "t" : 1358377964000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:12:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 142, "optime" : { "t" : 1358377965000, "i" : 7780 }, "optimeDate" : ISODate("2013-01-16T23:12:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:12:46.401 [conn9] end connection 127.0.0.2:59501 (2 connections now open) m31001| Wed Jan 16 18:12:46.401 [initandlisten] connection accepted from 127.0.0.2:33398 #11 (3 connections now open) m31000| Wed Jan 16 18:12:46.401 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:12:48.401 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:12:49.888 [conn15] end connection 127.0.0.4:51448 (5 connections now open) m31000| Wed Jan 16 18:12:49.888 [initandlisten] connection accepted from 127.0.0.4:58594 #16 (7 connections now open) STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:12:50.260 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:693893 346ms { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 149, "optime" : { "t" : 1358377970000, "i" : 791 }, "optimeDate" : ISODate("2013-01-16T23:12:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 138, "optime" : { "t" : 1358377965000, "i" : 7780 }, "optimeDate" : ISODate("2013-01-16T23:12:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 138, "optime" : { "t" : 1358377965000, "i" : 7780 }, "optimeDate" : ISODate("2013-01-16T23:12:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:49Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:50Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 148, "optime" : { "t" : 1358377969000, "i" : 2224 }, "optimeDate" : ISODate("2013-01-16T23:12:49Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 135, "optime" : { "t" : 1358377969000, "i" : 185 }, "optimeDate" : ISODate("2013-01-16T23:12:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 135, "optime" : { "t" : 1358377969000, "i" : 7690 }, "optimeDate" : ISODate("2013-01-16T23:12:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 135, "optime" : { "t" : 1358377965000, "i" : 7780 }, "optimeDate" : ISODate("2013-01-16T23:12:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:49Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 147, "optime" : { "t" : 1358377970000, "i" : 354 }, "optimeDate" : ISODate("2013-01-16T23:12:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:12:50.402 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:12:51.029 [rsMgr] not electing self, we are not freshest m31000| Wed Jan 16 18:12:51.795 [conn7] SocketException handling request, closing client connection: 9001 socket exception [2] server [127.0.0.3:46561] m31000| Wed Jan 16 18:12:52.402 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:12:54.007 [rsHealthPoll] replset info 127.0.0.2:31000 heartbeat failed, retrying m31000| Wed Jan 16 18:12:54.402 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 153, "optime" : { "t" : 1358377974000, "i" : 3872 }, "optimeDate" : ISODate("2013-01-16T23:12:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 142, "optime" : { "t" : 1358377973000, "i" : 16 }, "optimeDate" : ISODate("2013-01-16T23:12:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 142, "optime" : { "t" : 1358377973000, "i" : 863 }, "optimeDate" : ISODate("2013-01-16T23:12:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:53Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:54Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 152, "optime" : { "t" : 1358377973000, "i" : 17 }, "optimeDate" : ISODate("2013-01-16T23:12:53Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 139, "optime" : { "t" : 1358377970000, "i" : 791 }, "optimeDate" : ISODate("2013-01-16T23:12:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 139, "optime" : { "t" : 1358377973000, "i" : 5341 }, "optimeDate" : ISODate("2013-01-16T23:12:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 139, "optime" : { "t" : 1358377970000, "i" : 791 }, "optimeDate" : ISODate("2013-01-16T23:12:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:53Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 151, "optime" : { "t" : 1358377974000, "i" : 3872 }, "optimeDate" : ISODate("2013-01-16T23:12:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:12:55.896 [conn11] end connection 127.0.0.2:33901 (4 connections now open) m31002| Wed Jan 16 18:12:56.396 [initandlisten] connection accepted from 127.0.0.2:55516 #13 (5 connections now open) m31000| Wed Jan 16 18:12:56.403 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:12:57.030 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31000| Wed Jan 16 18:12:58.403 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 157, "optime" : { "t" : 1358377978000, "i" : 5056 }, "optimeDate" : ISODate("2013-01-16T23:12:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 146, "optime" : { "t" : 1358377977000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:12:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 146, "optime" : { "t" : 1358377974000, "i" : 3872 }, "optimeDate" : ISODate("2013-01-16T23:12:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:57Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:58Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 156, "optime" : { "t" : 1358377978000, "i" : 2592 }, "optimeDate" : ISODate("2013-01-16T23:12:58Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 143, "optime" : { "t" : 1358377974000, "i" : 3872 }, "optimeDate" : ISODate("2013-01-16T23:12:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:12:57Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:12:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 143, "optime" : { "t" : 1358377977000, "i" : 3837 }, "optimeDate" : ISODate("2013-01-16T23:12:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 143, "optime" : { "t" : 1358377974000, "i" : 3872 }, "optimeDate" : ISODate("2013-01-16T23:12:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 155, "optime" : { "t" : 1358377978000, "i" : 5056 }, "optimeDate" : ISODate("2013-01-16T23:12:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:13:00.386 [conn10] end connection 127.0.0.4:58713 (2 connections now open) m31000| Wed Jan 16 18:13:00.403 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:13:00.886 [initandlisten] connection accepted from 127.0.0.4:33317 #12 (3 connections now open) m31000| Wed Jan 16 18:13:02.404 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 161, "optime" : { "t" : 1358377982000, "i" : 6756 }, "optimeDate" : ISODate("2013-01-16T23:13:02Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 150, "optime" : { "t" : 1358377981000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:13:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 150, "optime" : { "t" : 1358377978000, "i" : 5056 }, "optimeDate" : ISODate("2013-01-16T23:12:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:01Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:02Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 160, "optime" : { "t" : 1358377982000, "i" : 3249 }, "optimeDate" : ISODate("2013-01-16T23:13:02Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 147, "optime" : { "t" : 1358377978000, "i" : 5056 }, "optimeDate" : ISODate("2013-01-16T23:12:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 147, "optime" : { "t" : 1358377981000, "i" : 2538 }, "optimeDate" : ISODate("2013-01-16T23:13:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 147, "optime" : { "t" : 1358377978000, "i" : 5056 }, "optimeDate" : ISODate("2013-01-16T23:12:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:01Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 159, "optime" : { "t" : 1358377982000, "i" : 6756 }, "optimeDate" : ISODate("2013-01-16T23:13:02Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:13:03.031 [conn12] end connection 127.0.0.3:49663 (4 connections now open) m31002| Wed Jan 16 18:13:03.031 [initandlisten] connection accepted from 127.0.0.3:44787 #14 (5 connections now open) m31001| Wed Jan 16 18:13:03.032 [rsMgr] not electing self, we are not freshest m31000| Wed Jan 16 18:13:04.404 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:13:06.404 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 165, "optime" : { "t" : 1358377986000, "i" : 8078 }, "optimeDate" : ISODate("2013-01-16T23:13:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 154, "optime" : { "t" : 1358377985000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:13:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 154, "optime" : { "t" : 1358377986000, "i" : 2807 }, "optimeDate" : ISODate("2013-01-16T23:13:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:06Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:12:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 164, "optime" : { "t" : 1358377986000, "i" : 2792 }, "optimeDate" : ISODate("2013-01-16T23:13:06Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 151, "optime" : { "t" : 1358377982000, "i" : 6756 }, "optimeDate" : ISODate("2013-01-16T23:13:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:05Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 151, "optime" : { "t" : 1358377985000, "i" : 884 }, "optimeDate" : ISODate("2013-01-16T23:13:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 151, "optime" : { "t" : 1358377982000, "i" : 6756 }, "optimeDate" : ISODate("2013-01-16T23:13:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 163, "optime" : { "t" : 1358377986000, "i" : 8078 }, "optimeDate" : ISODate("2013-01-16T23:13:06Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:13:07.007 [rsHealthPoll] replset info 127.0.0.2:31000 heartbeat failed, retrying m31001| Wed Jan 16 18:13:08.007 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31000| Wed Jan 16 18:13:08.405 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:13:10.405 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 170, "optime" : { "t" : 1358377991000, "i" : 935 }, "optimeDate" : ISODate("2013-01-16T23:13:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 159, "optime" : { "t" : 1358377989000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:13:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 159, "optime" : { "t" : 1358377986000, "i" : 8078 }, "optimeDate" : ISODate("2013-01-16T23:13:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:09Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:11Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 169, "optime" : { "t" : 1358377990000, "i" : 3493 }, "optimeDate" : ISODate("2013-01-16T23:13:10Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 156, "optime" : { "t" : 1358377991000, "i" : 48 }, "optimeDate" : ISODate("2013-01-16T23:13:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 156, "optime" : { "t" : 1358377986000, "i" : 8078 }, "optimeDate" : ISODate("2013-01-16T23:13:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 156, "optime" : { "t" : 1358377990000, "i" : 3493 }, "optimeDate" : ISODate("2013-01-16T23:13:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:11Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 168, "optime" : { "t" : 1358377991000, "i" : 284 }, "optimeDate" : ISODate("2013-01-16T23:13:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:13:12.405 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:13:13.034 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31000| Wed Jan 16 18:13:14.405 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 174, "optime" : { "t" : 1358377995000, "i" : 2744 }, "optimeDate" : ISODate("2013-01-16T23:13:15Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 163, "optime" : { "t" : 1358377991000, "i" : 935 }, "optimeDate" : ISODate("2013-01-16T23:13:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 163, "optime" : { "t" : 1358377991000, "i" : 935 }, "optimeDate" : ISODate("2013-01-16T23:13:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:15Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 173, "optime" : { "t" : 1358377994000, "i" : 2613 }, "optimeDate" : ISODate("2013-01-16T23:13:14Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 160, "optime" : { "t" : 1358377994000, "i" : 7117 }, "optimeDate" : ISODate("2013-01-16T23:13:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 160, "optime" : { "t" : 1358377991000, "i" : 935 }, "optimeDate" : ISODate("2013-01-16T23:13:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 160, "optime" : { "t" : 1358377991000, "i" : 935 }, "optimeDate" : ISODate("2013-01-16T23:13:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:15Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 172, "optime" : { "t" : 1358377995000, "i" : 2667 }, "optimeDate" : ISODate("2013-01-16T23:13:15Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:13:16.406 [conn11] end connection 127.0.0.2:33398 (2 connections now open) m31001| Wed Jan 16 18:13:16.406 [initandlisten] connection accepted from 127.0.0.2:39957 #13 (3 connections now open) m31000| Wed Jan 16 18:13:16.406 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:13:18.406 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:13:19.034 [rsMgr] not electing self, we are not freshest STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 178, "optime" : { "t" : 1358377999000, "i" : 4499 }, "optimeDate" : ISODate("2013-01-16T23:13:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 167, "optime" : { "t" : 1358377995000, "i" : 2744 }, "optimeDate" : ISODate("2013-01-16T23:13:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 167, "optime" : { "t" : 1358377998000, "i" : 4655 }, "optimeDate" : ISODate("2013-01-16T23:13:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:19Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 177, "optime" : { "t" : 1358377998000, "i" : 713 }, "optimeDate" : ISODate("2013-01-16T23:13:18Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 164, "optime" : { "t" : 1358377999000, "i" : 158 }, "optimeDate" : ISODate("2013-01-16T23:13:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 164, "optime" : { "t" : 1358377995000, "i" : 2744 }, "optimeDate" : ISODate("2013-01-16T23:13:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 164, "optime" : { "t" : 1358377995000, "i" : 2744 }, "optimeDate" : ISODate("2013-01-16T23:13:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:19Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 176, "optime" : { "t" : 1358377999000, "i" : 4391 }, "optimeDate" : ISODate("2013-01-16T23:13:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:13:19.893 [conn16] end connection 127.0.0.4:58594 (4 connections now open) m31000| Wed Jan 16 18:13:19.894 [initandlisten] connection accepted from 127.0.0.4:36427 #17 (5 connections now open) m31001| Wed Jan 16 18:13:20.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:13:20.407 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:13:22.407 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:13:22.977 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:330134 165ms STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:13:23.658 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:338111 169ms { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 182, "optime" : { "t" : 1358378003000, "i" : 6012 }, "optimeDate" : ISODate("2013-01-16T23:13:23Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 171, "optime" : { "t" : 1358377999000, "i" : 4499 }, "optimeDate" : ISODate("2013-01-16T23:13:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 171, "optime" : { "t" : 1358377999000, "i" : 4499 }, "optimeDate" : ISODate("2013-01-16T23:13:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:21Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:23Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 181, "optime" : { "t" : 1358378003000, "i" : 1014 }, "optimeDate" : ISODate("2013-01-16T23:13:23Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 168, "optime" : { "t" : 1358378003000, "i" : 518 }, "optimeDate" : ISODate("2013-01-16T23:13:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:23Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 168, "optime" : { "t" : 1358377999000, "i" : 4499 }, "optimeDate" : ISODate("2013-01-16T23:13:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 168, "optime" : { "t" : 1358377999000, "i" : 4499 }, "optimeDate" : ISODate("2013-01-16T23:13:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 180, "optime" : { "t" : 1358378003000, "i" : 6012 }, "optimeDate" : ISODate("2013-01-16T23:13:23Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:13:24.407 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:13:25.035 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31000| Wed Jan 16 18:13:26.408 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 186, "optime" : { "t" : 1358378007000, "i" : 7140 }, "optimeDate" : ISODate("2013-01-16T23:13:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 175, "optime" : { "t" : 1358378003000, "i" : 6012 }, "optimeDate" : ISODate("2013-01-16T23:13:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 175, "optime" : { "t" : 1358378003000, "i" : 6012 }, "optimeDate" : ISODate("2013-01-16T23:13:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:27Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 185, "optime" : { "t" : 1358378007000, "i" : 3938 }, "optimeDate" : ISODate("2013-01-16T23:13:27Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 172, "optime" : { "t" : 1358378007000, "i" : 6 }, "optimeDate" : ISODate("2013-01-16T23:13:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 172, "optime" : { "t" : 1358378003000, "i" : 6012 }, "optimeDate" : ISODate("2013-01-16T23:13:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 172, "optime" : { "t" : 1358378003000, "i" : 6012 }, "optimeDate" : ISODate("2013-01-16T23:13:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:27Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 184, "optime" : { "t" : 1358378007000, "i" : 7140 }, "optimeDate" : ISODate("2013-01-16T23:13:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:13:28.408 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:13:30.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:13:30.409 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:13:31.036 [rsMgr] not electing self, we are not freshest STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 191, "optime" : { "t" : 1358378012000, "i" : 94 }, "optimeDate" : ISODate("2013-01-16T23:13:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 180, "optime" : { "t" : 1358378007000, "i" : 7140 }, "optimeDate" : ISODate("2013-01-16T23:13:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 180, "optime" : { "t" : 1358378011000, "i" : 3658 }, "optimeDate" : ISODate("2013-01-16T23:13:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:31Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:32Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 190, "optime" : { "t" : 1358378011000, "i" : 4953 }, "optimeDate" : ISODate("2013-01-16T23:13:31Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 177, "optime" : { "t" : 1358378011000, "i" : 578 }, "optimeDate" : ISODate("2013-01-16T23:13:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 177, "optime" : { "t" : 1358378011000, "i" : 8028 }, "optimeDate" : ISODate("2013-01-16T23:13:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 177, "optime" : { "t" : 1358378007000, "i" : 7140 }, "optimeDate" : ISODate("2013-01-16T23:13:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:31Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 189, "optime" : { "t" : 1358378012000, "i" : 13 }, "optimeDate" : ISODate("2013-01-16T23:13:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:13:32.409 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:13:33.036 [conn14] end connection 127.0.0.3:44787 (4 connections now open) m31002| Wed Jan 16 18:13:33.036 [initandlisten] connection accepted from 127.0.0.3:50041 #15 (5 connections now open) m31002| Wed Jan 16 18:13:33.901 [conn13] end connection 127.0.0.2:55516 (4 connections now open) m31002| Wed Jan 16 18:13:34.402 [initandlisten] connection accepted from 127.0.0.2:56720 #16 (5 connections now open) m31000| Wed Jan 16 18:13:34.410 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 195, "optime" : { "t" : 1358378016000, "i" : 1553 }, "optimeDate" : ISODate("2013-01-16T23:13:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 184, "optime" : { "t" : 1358378012000, "i" : 94 }, "optimeDate" : ISODate("2013-01-16T23:13:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 184, "optime" : { "t" : 1358378012000, "i" : 94 }, "optimeDate" : ISODate("2013-01-16T23:13:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:35Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:36Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 194, "optime" : { "t" : 1358378015000, "i" : 4536 }, "optimeDate" : ISODate("2013-01-16T23:13:35Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 181, "optime" : { "t" : 1358378015000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:13:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:35Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 181, "optime" : { "t" : 1358378015000, "i" : 7462 }, "optimeDate" : ISODate("2013-01-16T23:13:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 181, "optime" : { "t" : 1358378015000, "i" : 4536 }, "optimeDate" : ISODate("2013-01-16T23:13:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 193, "optime" : { "t" : 1358378016000, "i" : 1251 }, "optimeDate" : ISODate("2013-01-16T23:13:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:13:36.410 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:13:37.038 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31001| Wed Jan 16 18:13:38.391 [conn12] end connection 127.0.0.4:33317 (2 connections now open) m31000| Wed Jan 16 18:13:38.411 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:13:38.891 [initandlisten] connection accepted from 127.0.0.4:50000 #14 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 199, "optime" : { "t" : 1358378020000, "i" : 2730 }, "optimeDate" : ISODate("2013-01-16T23:13:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 188, "optime" : { "t" : 1358378016000, "i" : 1553 }, "optimeDate" : ISODate("2013-01-16T23:13:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 188, "optime" : { "t" : 1358378019000, "i" : 1854 }, "optimeDate" : ISODate("2013-01-16T23:13:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:39Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:40Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 198, "optime" : { "t" : 1358378019000, "i" : 3321 }, "optimeDate" : ISODate("2013-01-16T23:13:39Z"), "errmsg" : "syncing to: 127.0.0.4:31002", "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 185, "optime" : { "t" : 1358378016000, "i" : 1553 }, "optimeDate" : ISODate("2013-01-16T23:13:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 185, "optime" : { "t" : 1358378019000, "i" : 6304 }, "optimeDate" : ISODate("2013-01-16T23:13:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 185, "optime" : { "t" : 1358378016000, "i" : 1553 }, "optimeDate" : ISODate("2013-01-16T23:13:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:39Z"), "pingMs" : 500, "lastHeartbeatMessage" : "syncing to: 127.0.0.4:31002" }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 197, "optime" : { "t" : 1358378020000, "i" : 2627 }, "optimeDate" : ISODate("2013-01-16T23:13:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:13:40.411 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:13:42.411 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:13:43.039 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31000| Wed Jan 16 18:13:43.907 [FileAllocator] allocating new datafile /data/db/testReplSet-0/foo.2, filling with zeroes... STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:13:44.412 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:13:44.426 [FileAllocator] done allocating datafile /data/db/testReplSet-0/foo.2, size: 64MB, took 0.518 secs m31000| Wed Jan 16 18:13:44.426 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:519628 519ms m31002| Wed Jan 16 18:13:44.427 [FileAllocator] allocating new datafile /data/db/testReplSet-2/foo.2, filling with zeroes... { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 203, "optime" : { "t" : 1358378024000, "i" : 5445 }, "optimeDate" : ISODate("2013-01-16T23:13:44Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 192, "optime" : { "t" : 1358378023000, "i" : 1829 }, "optimeDate" : ISODate("2013-01-16T23:13:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 192, "optime" : { "t" : 1358378023000, "i" : 4555 }, "optimeDate" : ISODate("2013-01-16T23:13:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:44Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 202, "optime" : { "t" : 1358378023000, "i" : 1829 }, "optimeDate" : ISODate("2013-01-16T23:13:43Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 189, "optime" : { "t" : 1358378020000, "i" : 2730 }, "optimeDate" : ISODate("2013-01-16T23:13:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:43Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 189, "optime" : { "t" : 1358378023000, "i" : 4445 }, "optimeDate" : ISODate("2013-01-16T23:13:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 189, "optime" : { "t" : 1358378023000, "i" : 1829 }, "optimeDate" : ISODate("2013-01-16T23:13:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 201, "optime" : { "t" : 1358378023000, "i" : 4555 }, "optimeDate" : ISODate("2013-01-16T23:13:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:13:44.961 [FileAllocator] done allocating datafile /data/db/testReplSet-2/foo.2, size: 64MB, took 0.533 secs m31002| Wed Jan 16 18:13:44.964 [conn8] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358377825000|1 } } cursorid:331275280870700 ntoreturn:0 keyUpdates:0 locks(micros) r:2566 nreturned:2727 reslen:264539 1360ms m31001| Wed Jan 16 18:13:45.021 [FileAllocator] allocating new datafile /data/db/testReplSet-1/foo.2, filling with zeroes... m31001| Wed Jan 16 18:13:45.545 [FileAllocator] done allocating datafile /data/db/testReplSet-1/foo.2, size: 64MB, took 0.524 secs m31001| Wed Jan 16 18:13:46.412 [conn13] end connection 127.0.0.2:39957 (2 connections now open) m31001| Wed Jan 16 18:13:46.412 [initandlisten] connection accepted from 127.0.0.2:40875 #15 (3 connections now open) m31000| Wed Jan 16 18:13:46.412 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:13:48.413 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 207, "optime" : { "t" : 1358378028000, "i" : 7068 }, "optimeDate" : ISODate("2013-01-16T23:13:48Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 196, "optime" : { "t" : 1358378027000, "i" : 2316 }, "optimeDate" : ISODate("2013-01-16T23:13:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 196, "optime" : { "t" : 1358378024000, "i" : 5445 }, "optimeDate" : ISODate("2013-01-16T23:13:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:47Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:48Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 206, "optime" : { "t" : 1358378027000, "i" : 2316 }, "optimeDate" : ISODate("2013-01-16T23:13:47Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 193, "optime" : { "t" : 1358378024000, "i" : 5445 }, "optimeDate" : ISODate("2013-01-16T23:13:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 193, "optime" : { "t" : 1358378027000, "i" : 1718 }, "optimeDate" : ISODate("2013-01-16T23:13:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 193, "optime" : { "t" : 1358378024000, "i" : 5445 }, "optimeDate" : ISODate("2013-01-16T23:13:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 205, "optime" : { "t" : 1358378028000, "i" : 7068 }, "optimeDate" : ISODate("2013-01-16T23:13:48Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:13:49.039 [rsMgr] not electing self, we are not freshest m31000| Wed Jan 16 18:13:49.899 [conn17] end connection 127.0.0.4:36427 (4 connections now open) m31000| Wed Jan 16 18:13:49.899 [initandlisten] connection accepted from 127.0.0.4:45785 #18 (5 connections now open) m31000| Wed Jan 16 18:13:50.413 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:13:52.006 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:13:52.413 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 212, "optime" : { "t" : 1358378033000, "i" : 423 }, "optimeDate" : ISODate("2013-01-16T23:13:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 201, "optime" : { "t" : 1358378032000, "i" : 278 }, "optimeDate" : ISODate("2013-01-16T23:13:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 201, "optime" : { "t" : 1358378031000, "i" : 431 }, "optimeDate" : ISODate("2013-01-16T23:13:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:53Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 211, "optime" : { "t" : 1358378032000, "i" : 278 }, "optimeDate" : ISODate("2013-01-16T23:13:52Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 198, "optime" : { "t" : 1358378028000, "i" : 7068 }, "optimeDate" : ISODate("2013-01-16T23:13:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:51Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 198, "optime" : { "t" : 1358378031000, "i" : 407 }, "optimeDate" : ISODate("2013-01-16T23:13:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:51Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 198, "optime" : { "t" : 1358378028000, "i" : 7068 }, "optimeDate" : ISODate("2013-01-16T23:13:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 210, "optime" : { "t" : 1358378033000, "i" : 251 }, "optimeDate" : ISODate("2013-01-16T23:13:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:13:54.414 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:13:55.040 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31000| Wed Jan 16 18:13:56.414 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 216, "optime" : { "t" : 1358378037000, "i" : 1236 }, "optimeDate" : ISODate("2013-01-16T23:13:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 205, "optime" : { "t" : 1358378036000, "i" : 4 }, "optimeDate" : ISODate("2013-01-16T23:13:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 205, "optime" : { "t" : 1358378036000, "i" : 7640 }, "optimeDate" : ISODate("2013-01-16T23:13:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:57Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 215, "optime" : { "t" : 1358378036000, "i" : 4 }, "optimeDate" : ISODate("2013-01-16T23:13:56Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 202, "optime" : { "t" : 1358378037000, "i" : 219 }, "optimeDate" : ISODate("2013-01-16T23:13:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:13:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 202, "optime" : { "t" : 1358378033000, "i" : 423 }, "optimeDate" : ISODate("2013-01-16T23:13:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 202, "optime" : { "t" : 1358378036000, "i" : 4 }, "optimeDate" : ISODate("2013-01-16T23:13:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 214, "optime" : { "t" : 1358378037000, "i" : 733 }, "optimeDate" : ISODate("2013-01-16T23:13:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:13:58.414 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:14:00.358 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:356870 178ms m31000| Wed Jan 16 18:14:00.414 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:14:01.041 [rsMgr] not electing self, we are not freshest STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 220, "optime" : { "t" : 1358378041000, "i" : 1822 }, "optimeDate" : ISODate("2013-01-16T23:14:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 209, "optime" : { "t" : 1358378037000, "i" : 1236 }, "optimeDate" : ISODate("2013-01-16T23:13:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 209, "optime" : { "t" : 1358378037000, "i" : 1236 }, "optimeDate" : ISODate("2013-01-16T23:13:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:13:59Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:01Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 219, "optime" : { "t" : 1358378040000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:14:00Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 206, "optime" : { "t" : 1358378041000, "i" : 7 }, "optimeDate" : ISODate("2013-01-16T23:14:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 206, "optime" : { "t" : 1358378037000, "i" : 1236 }, "optimeDate" : ISODate("2013-01-16T23:13:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 206, "optime" : { "t" : 1358378037000, "i" : 1236 }, "optimeDate" : ISODate("2013-01-16T23:13:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 218, "optime" : { "t" : 1358378041000, "i" : 1236 }, "optimeDate" : ISODate("2013-01-16T23:14:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:14:02.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:14:02.415 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:14:03.041 [conn15] end connection 127.0.0.3:50041 (4 connections now open) m31002| Wed Jan 16 18:14:03.042 [initandlisten] connection accepted from 127.0.0.3:42991 #17 (5 connections now open) m31000| Wed Jan 16 18:14:04.415 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 224, "optime" : { "t" : 1358378045000, "i" : 3428 }, "optimeDate" : ISODate("2013-01-16T23:14:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 213, "optime" : { "t" : 1358378041000, "i" : 1822 }, "optimeDate" : ISODate("2013-01-16T23:14:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 213, "optime" : { "t" : 1358378044000, "i" : 1637 }, "optimeDate" : ISODate("2013-01-16T23:14:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:05Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 223, "optime" : { "t" : 1358378045000, "i" : 857 }, "optimeDate" : ISODate("2013-01-16T23:14:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 210, "optime" : { "t" : 1358378045000, "i" : 621 }, "optimeDate" : ISODate("2013-01-16T23:14:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 210, "optime" : { "t" : 1358378041000, "i" : 1822 }, "optimeDate" : ISODate("2013-01-16T23:14:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 210, "optime" : { "t" : 1358378041000, "i" : 1822 }, "optimeDate" : ISODate("2013-01-16T23:14:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:05Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 222, "optime" : { "t" : 1358378045000, "i" : 3428 }, "optimeDate" : ISODate("2013-01-16T23:14:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:14:06.415 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:14:07.043 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31000| Wed Jan 16 18:14:08.416 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 228, "optime" : { "t" : 1358378049000, "i" : 4481 }, "optimeDate" : ISODate("2013-01-16T23:14:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 217, "optime" : { "t" : 1358378045000, "i" : 3428 }, "optimeDate" : ISODate("2013-01-16T23:14:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 217, "optime" : { "t" : 1358378049000, "i" : 3527 }, "optimeDate" : ISODate("2013-01-16T23:14:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:09Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:13:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 227, "optime" : { "t" : 1358378049000, "i" : 426 }, "optimeDate" : ISODate("2013-01-16T23:14:09Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 214, "optime" : { "t" : 1358378049000, "i" : 279 }, "optimeDate" : ISODate("2013-01-16T23:14:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 214, "optime" : { "t" : 1358378045000, "i" : 3428 }, "optimeDate" : ISODate("2013-01-16T23:14:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:09Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 214, "optime" : { "t" : 1358378045000, "i" : 3428 }, "optimeDate" : ISODate("2013-01-16T23:14:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:09Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 226, "optime" : { "t" : 1358378049000, "i" : 4481 }, "optimeDate" : ISODate("2013-01-16T23:14:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:14:10.416 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:14:11.907 [conn16] end connection 127.0.0.2:56720 (4 connections now open) m31001| Wed Jan 16 18:14:12.007 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31002| Wed Jan 16 18:14:12.407 [initandlisten] connection accepted from 127.0.0.2:37226 #18 (5 connections now open) m31000| Wed Jan 16 18:14:12.416 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 232, "optime" : { "t" : 1358378053000, "i" : 6208 }, "optimeDate" : ISODate("2013-01-16T23:14:13Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 221, "optime" : { "t" : 1358378049000, "i" : 4481 }, "optimeDate" : ISODate("2013-01-16T23:14:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 221, "optime" : { "t" : 1358378049000, "i" : 4481 }, "optimeDate" : ISODate("2013-01-16T23:14:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:13Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 231, "optime" : { "t" : 1358378053000, "i" : 1170 }, "optimeDate" : ISODate("2013-01-16T23:14:13Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 218, "optime" : { "t" : 1358378053000, "i" : 546 }, "optimeDate" : ISODate("2013-01-16T23:14:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 218, "optime" : { "t" : 1358378049000, "i" : 4481 }, "optimeDate" : ISODate("2013-01-16T23:14:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 218, "optime" : { "t" : 1358378049000, "i" : 4481 }, "optimeDate" : ISODate("2013-01-16T23:14:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:13Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 230, "optime" : { "t" : 1358378053000, "i" : 6208 }, "optimeDate" : ISODate("2013-01-16T23:14:13Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:14:14.417 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:14:16.396 [conn14] end connection 127.0.0.4:50000 (2 connections now open) m31001| Wed Jan 16 18:14:16.417 [conn15] end connection 127.0.0.2:40875 (1 connection now open) m31001| Wed Jan 16 18:14:16.417 [initandlisten] connection accepted from 127.0.0.2:41188 #16 (2 connections now open) m31000| Wed Jan 16 18:14:16.418 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:14:16.897 [initandlisten] connection accepted from 127.0.0.4:38209 #17 (3 connections now open) m31000| Wed Jan 16 18:14:16.912 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:338090 169ms m31001| Wed Jan 16 18:14:17.044 [rsMgr] not electing self, we are not freshest STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 236, "optime" : { "t" : 1358378057000, "i" : 7353 }, "optimeDate" : ISODate("2013-01-16T23:14:17Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 225, "optime" : { "t" : 1358378053000, "i" : 6208 }, "optimeDate" : ISODate("2013-01-16T23:14:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 225, "optime" : { "t" : 1358378057000, "i" : 4225 }, "optimeDate" : ISODate("2013-01-16T23:14:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:17Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 235, "optime" : { "t" : 1358378057000, "i" : 1559 }, "optimeDate" : ISODate("2013-01-16T23:14:17Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 222, "optime" : { "t" : 1358378057000, "i" : 313 }, "optimeDate" : ISODate("2013-01-16T23:14:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 222, "optime" : { "t" : 1358378053000, "i" : 6208 }, "optimeDate" : ISODate("2013-01-16T23:14:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:17Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 222, "optime" : { "t" : 1358378053000, "i" : 6208 }, "optimeDate" : ISODate("2013-01-16T23:14:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 234, "optime" : { "t" : 1358378057000, "i" : 7253 }, "optimeDate" : ISODate("2013-01-16T23:14:17Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:14:18.418 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:14:19.905 [conn18] end connection 127.0.0.4:45785 (4 connections now open) m31000| Wed Jan 16 18:14:19.905 [initandlisten] connection accepted from 127.0.0.4:59619 #19 (5 connections now open) m31000| Wed Jan 16 18:14:20.419 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:14:21.224 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:342993 171ms m31000| Wed Jan 16 18:14:21.432 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:206051 103ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 240, "optime" : { "t" : 1358378061000, "i" : 8553 }, "optimeDate" : ISODate("2013-01-16T23:14:21Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 229, "optime" : { "t" : 1358378057000, "i" : 7353 }, "optimeDate" : ISODate("2013-01-16T23:14:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 229, "optime" : { "t" : 1358378057000, "i" : 7353 }, "optimeDate" : ISODate("2013-01-16T23:14:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:21Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:21Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 239, "optime" : { "t" : 1358378061000, "i" : 358 }, "optimeDate" : ISODate("2013-01-16T23:14:21Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 226, "optime" : { "t" : 1358378061000, "i" : 298 }, "optimeDate" : ISODate("2013-01-16T23:14:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:21Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 226, "optime" : { "t" : 1358378061000, "i" : 8055 }, "optimeDate" : ISODate("2013-01-16T23:14:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 226, "optime" : { "t" : 1358378061000, "i" : 358 }, "optimeDate" : ISODate("2013-01-16T23:14:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 238, "optime" : { "t" : 1358378061000, "i" : 8553 }, "optimeDate" : ISODate("2013-01-16T23:14:21Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:14:22.419 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:14:23.045 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31001| Wed Jan 16 18:14:24.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:14:24.419 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:14:25.632 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:204303 102ms STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:14:26.201 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:277586 138ms { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 245, "optime" : { "t" : 1358378066000, "i" : 826 }, "optimeDate" : ISODate("2013-01-16T23:14:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 234, "optime" : { "t" : 1358378061000, "i" : 8553 }, "optimeDate" : ISODate("2013-01-16T23:14:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 234, "optime" : { "t" : 1358378061000, "i" : 8553 }, "optimeDate" : ISODate("2013-01-16T23:14:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:25Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:26Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 244, "optime" : { "t" : 1358378065000, "i" : 1592 }, "optimeDate" : ISODate("2013-01-16T23:14:25Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 231, "optime" : { "t" : 1358378065000, "i" : 165 }, "optimeDate" : ISODate("2013-01-16T23:14:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 231, "optime" : { "t" : 1358378065000, "i" : 7890 }, "optimeDate" : ISODate("2013-01-16T23:14:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 231, "optime" : { "t" : 1358378061000, "i" : 8553 }, "optimeDate" : ISODate("2013-01-16T23:14:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 243, "optime" : { "t" : 1358378066000, "i" : 826 }, "optimeDate" : ISODate("2013-01-16T23:14:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:14:26.419 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:14:28.420 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:14:29.046 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 249, "optime" : { "t" : 1358378070000, "i" : 3167 }, "optimeDate" : ISODate("2013-01-16T23:14:30Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 238, "optime" : { "t" : 1358378066000, "i" : 826 }, "optimeDate" : ISODate("2013-01-16T23:14:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 238, "optime" : { "t" : 1358378069000, "i" : 5983 }, "optimeDate" : ISODate("2013-01-16T23:14:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:30Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 248, "optime" : { "t" : 1358378069000, "i" : 127 }, "optimeDate" : ISODate("2013-01-16T23:14:29Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 235, "optime" : { "t" : 1358378066000, "i" : 826 }, "optimeDate" : ISODate("2013-01-16T23:14:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:29Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 235, "optime" : { "t" : 1358378069000, "i" : 6110 }, "optimeDate" : ISODate("2013-01-16T23:14:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:29Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 235, "optime" : { "t" : 1358378069000, "i" : 127 }, "optimeDate" : ISODate("2013-01-16T23:14:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 247, "optime" : { "t" : 1358378070000, "i" : 2858 }, "optimeDate" : ISODate("2013-01-16T23:14:30Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:14:30.420 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:14:32.420 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:14:33.047 [conn17] end connection 127.0.0.3:42991 (4 connections now open) m31002| Wed Jan 16 18:14:33.047 [initandlisten] connection accepted from 127.0.0.3:54631 #19 (5 connections now open) m31001| Wed Jan 16 18:14:34.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:14:34.421 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 253, "optime" : { "t" : 1358378074000, "i" : 4630 }, "optimeDate" : ISODate("2013-01-16T23:14:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 242, "optime" : { "t" : 1358378073000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:14:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 242, "optime" : { "t" : 1358378070000, "i" : 3167 }, "optimeDate" : ISODate("2013-01-16T23:14:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:33Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:34Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 252, "optime" : { "t" : 1358378074000, "i" : 2232 }, "optimeDate" : ISODate("2013-01-16T23:14:34Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 239, "optime" : { "t" : 1358378070000, "i" : 3167 }, "optimeDate" : ISODate("2013-01-16T23:14:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:34Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 239, "optime" : { "t" : 1358378073000, "i" : 4475 }, "optimeDate" : ISODate("2013-01-16T23:14:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 239, "optime" : { "t" : 1358378073000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:14:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 251, "optime" : { "t" : 1358378074000, "i" : 4458 }, "optimeDate" : ISODate("2013-01-16T23:14:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:14:35.048 [rsMgr] not electing self, we are not freshest m31000| Wed Jan 16 18:14:36.421 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:14:38.421 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 257, "optime" : { "t" : 1358378078000, "i" : 6507 }, "optimeDate" : ISODate("2013-01-16T23:14:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 246, "optime" : { "t" : 1358378077000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:14:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 246, "optime" : { "t" : 1358378074000, "i" : 4630 }, "optimeDate" : ISODate("2013-01-16T23:14:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:37Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:38Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 256, "optime" : { "t" : 1358378078000, "i" : 2464 }, "optimeDate" : ISODate("2013-01-16T23:14:38Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 243, "optime" : { "t" : 1358378074000, "i" : 4630 }, "optimeDate" : ISODate("2013-01-16T23:14:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 243, "optime" : { "t" : 1358378077000, "i" : 2559 }, "optimeDate" : ISODate("2013-01-16T23:14:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 243, "optime" : { "t" : 1358378074000, "i" : 4630 }, "optimeDate" : ISODate("2013-01-16T23:14:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 255, "optime" : { "t" : 1358378078000, "i" : 6507 }, "optimeDate" : ISODate("2013-01-16T23:14:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:14:40.422 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:14:41.049 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31000| Wed Jan 16 18:14:42.422 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 262, "optime" : { "t" : 1358378083000, "i" : 493 }, "optimeDate" : ISODate("2013-01-16T23:14:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 251, "optime" : { "t" : 1358378081000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:14:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 251, "optime" : { "t" : 1358378082000, "i" : 3476 }, "optimeDate" : ISODate("2013-01-16T23:14:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:43Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 261, "optime" : { "t" : 1358378082000, "i" : 2512 }, "optimeDate" : ISODate("2013-01-16T23:14:42Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 248, "optime" : { "t" : 1358378078000, "i" : 6507 }, "optimeDate" : ISODate("2013-01-16T23:14:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:41Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 248, "optime" : { "t" : 1358378081000, "i" : 361 }, "optimeDate" : ISODate("2013-01-16T23:14:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 248, "optime" : { "t" : 1358378078000, "i" : 6507 }, "optimeDate" : ISODate("2013-01-16T23:14:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 260, "optime" : { "t" : 1358378083000, "i" : 444 }, "optimeDate" : ISODate("2013-01-16T23:14:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:14:44.422 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:14:46.422 [conn16] end connection 127.0.0.2:41188 (2 connections now open) m31001| Wed Jan 16 18:14:46.422 [initandlisten] connection accepted from 127.0.0.2:53849 #18 (3 connections now open) m31000| Wed Jan 16 18:14:46.422 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:14:47.050 [rsMgr] not electing self, we are not freshest STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 266, "optime" : { "t" : 1358378087000, "i" : 1576 }, "optimeDate" : ISODate("2013-01-16T23:14:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 255, "optime" : { "t" : 1358378086000, "i" : 2768 }, "optimeDate" : ISODate("2013-01-16T23:14:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 255, "optime" : { "t" : 1358378083000, "i" : 493 }, "optimeDate" : ISODate("2013-01-16T23:14:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:45Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:47Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 265, "optime" : { "t" : 1358378086000, "i" : 2768 }, "optimeDate" : ISODate("2013-01-16T23:14:46Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 252, "optime" : { "t" : 1358378087000, "i" : 800 }, "optimeDate" : ISODate("2013-01-16T23:14:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 252, "optime" : { "t" : 1358378083000, "i" : 493 }, "optimeDate" : ISODate("2013-01-16T23:14:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 252, "optime" : { "t" : 1358378086000, "i" : 2768 }, "optimeDate" : ISODate("2013-01-16T23:14:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 264, "optime" : { "t" : 1358378087000, "i" : 1576 }, "optimeDate" : ISODate("2013-01-16T23:14:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:14:48.423 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:14:49.910 [conn19] end connection 127.0.0.4:59619 (4 connections now open) m31000| Wed Jan 16 18:14:49.910 [initandlisten] connection accepted from 127.0.0.4:39630 #20 (5 connections now open) m31002| Wed Jan 16 18:14:49.912 [conn18] end connection 127.0.0.2:37226 (4 connections now open) m31002| Wed Jan 16 18:14:50.412 [initandlisten] connection accepted from 127.0.0.2:54600 #20 (5 connections now open) m31000| Wed Jan 16 18:14:50.423 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:14:51.145 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:226707 113ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 270, "optime" : { "t" : 1358378091000, "i" : 2744 }, "optimeDate" : ISODate("2013-01-16T23:14:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 259, "optime" : { "t" : 1358378090000, "i" : 593 }, "optimeDate" : ISODate("2013-01-16T23:14:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 259, "optime" : { "t" : 1358378090000, "i" : 1855 }, "optimeDate" : ISODate("2013-01-16T23:14:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:51Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 269, "optime" : { "t" : 1358378090000, "i" : 1855 }, "optimeDate" : ISODate("2013-01-16T23:14:50Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 256, "optime" : { "t" : 1358378091000, "i" : 101 }, "optimeDate" : ISODate("2013-01-16T23:14:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 256, "optime" : { "t" : 1358378087000, "i" : 1576 }, "optimeDate" : ISODate("2013-01-16T23:14:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 256, "optime" : { "t" : 1358378087000, "i" : 1576 }, "optimeDate" : ISODate("2013-01-16T23:14:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 268, "optime" : { "t" : 1358378091000, "i" : 2744 }, "optimeDate" : ISODate("2013-01-16T23:14:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:14:52.424 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:14:53.051 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31001| Wed Jan 16 18:14:54.402 [conn17] end connection 127.0.0.4:38209 (2 connections now open) m31000| Wed Jan 16 18:14:54.424 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:14:54.902 [initandlisten] connection accepted from 127.0.0.4:54608 #19 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 274, "optime" : { "t" : 1358378095000, "i" : 3615 }, "optimeDate" : ISODate("2013-01-16T23:14:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 263, "optime" : { "t" : 1358378094000, "i" : 277 }, "optimeDate" : ISODate("2013-01-16T23:14:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 263, "optime" : { "t" : 1358378091000, "i" : 2744 }, "optimeDate" : ISODate("2013-01-16T23:14:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:53Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:55Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 273, "optime" : { "t" : 1358378094000, "i" : 277 }, "optimeDate" : ISODate("2013-01-16T23:14:54Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 260, "optime" : { "t" : 1358378095000, "i" : 237 }, "optimeDate" : ISODate("2013-01-16T23:14:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 260, "optime" : { "t" : 1358378091000, "i" : 2744 }, "optimeDate" : ISODate("2013-01-16T23:14:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 260, "optime" : { "t" : 1358378094000, "i" : 277 }, "optimeDate" : ISODate("2013-01-16T23:14:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 272, "optime" : { "t" : 1358378095000, "i" : 3002 }, "optimeDate" : ISODate("2013-01-16T23:14:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:14:56.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:14:56.424 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:14:58.424 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:14:59.052 [rsMgr] not electing self, we are not freshest STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 278, "optime" : { "t" : 1358378099000, "i" : 4889 }, "optimeDate" : ISODate("2013-01-16T23:14:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 267, "optime" : { "t" : 1358378095000, "i" : 3615 }, "optimeDate" : ISODate("2013-01-16T23:14:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 267, "optime" : { "t" : 1358378095000, "i" : 3615 }, "optimeDate" : ISODate("2013-01-16T23:14:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:59Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 277, "optime" : { "t" : 1358378098000, "i" : 158 }, "optimeDate" : ISODate("2013-01-16T23:14:58Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 264, "optime" : { "t" : 1358378099000, "i" : 370 }, "optimeDate" : ISODate("2013-01-16T23:14:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:14:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 264, "optime" : { "t" : 1358378095000, "i" : 3615 }, "optimeDate" : ISODate("2013-01-16T23:14:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:57Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 264, "optime" : { "t" : 1358378095000, "i" : 3615 }, "optimeDate" : ISODate("2013-01-16T23:14:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:14:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 276, "optime" : { "t" : 1358378099000, "i" : 4397 }, "optimeDate" : ISODate("2013-01-16T23:14:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:15:00.425 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:15:02.425 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:15:03.052 [conn19] end connection 127.0.0.3:54631 (4 connections now open) m31002| Wed Jan 16 18:15:03.052 [initandlisten] connection accepted from 127.0.0.3:39876 #21 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 282, "optime" : { "t" : 1358378103000, "i" : 6410 }, "optimeDate" : ISODate("2013-01-16T23:15:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 271, "optime" : { "t" : 1358378099000, "i" : 4889 }, "optimeDate" : ISODate("2013-01-16T23:14:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 271, "optime" : { "t" : 1358378102000, "i" : 2490 }, "optimeDate" : ISODate("2013-01-16T23:15:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:03Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 281, "optime" : { "t" : 1358378103000, "i" : 4394 }, "optimeDate" : ISODate("2013-01-16T23:15:03Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 268, "optime" : { "t" : 1358378103000, "i" : 292 }, "optimeDate" : ISODate("2013-01-16T23:15:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 268, "optime" : { "t" : 1358378099000, "i" : 4889 }, "optimeDate" : ISODate("2013-01-16T23:14:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 268, "optime" : { "t" : 1358378099000, "i" : 4889 }, "optimeDate" : ISODate("2013-01-16T23:14:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:03Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 280, "optime" : { "t" : 1358378103000, "i" : 5684 }, "optimeDate" : ISODate("2013-01-16T23:15:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:15:04.426 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:15:05.053 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31001| Wed Jan 16 18:15:06.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:15:06.426 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 286, "optime" : { "t" : 1358378107000, "i" : 7361 }, "optimeDate" : ISODate("2013-01-16T23:15:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 275, "optime" : { "t" : 1358378103000, "i" : 6410 }, "optimeDate" : ISODate("2013-01-16T23:15:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 275, "optime" : { "t" : 1358378103000, "i" : 6410 }, "optimeDate" : ISODate("2013-01-16T23:15:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:05Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:07Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 285, "optime" : { "t" : 1358378107000, "i" : 4304 }, "optimeDate" : ISODate("2013-01-16T23:15:07Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 272, "optime" : { "t" : 1358378107000, "i" : 443 }, "optimeDate" : ISODate("2013-01-16T23:15:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:07Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 272, "optime" : { "t" : 1358378103000, "i" : 6410 }, "optimeDate" : ISODate("2013-01-16T23:15:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 272, "optime" : { "t" : 1358378103000, "i" : 6410 }, "optimeDate" : ISODate("2013-01-16T23:15:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 284, "optime" : { "t" : 1358378107000, "i" : 6995 }, "optimeDate" : ISODate("2013-01-16T23:15:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:15:08.427 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:15:10.427 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:15:11.054 [rsMgr] not electing self, we are not freshest STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 291, "optime" : { "t" : 1358378112000, "i" : 38 }, "optimeDate" : ISODate("2013-01-16T23:15:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 280, "optime" : { "t" : 1358378107000, "i" : 7361 }, "optimeDate" : ISODate("2013-01-16T23:15:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 280, "optime" : { "t" : 1358378107000, "i" : 7361 }, "optimeDate" : ISODate("2013-01-16T23:15:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:11Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:12Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:14:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 290, "optime" : { "t" : 1358378111000, "i" : 4446 }, "optimeDate" : ISODate("2013-01-16T23:15:11Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 277, "optime" : { "t" : 1358378111000, "i" : 206 }, "optimeDate" : ISODate("2013-01-16T23:15:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 277, "optime" : { "t" : 1358378111000, "i" : 7777 }, "optimeDate" : ISODate("2013-01-16T23:15:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 277, "optime" : { "t" : 1358378107000, "i" : 7361 }, "optimeDate" : ISODate("2013-01-16T23:15:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 289, "optime" : { "t" : 1358378112000, "i" : 38 }, "optimeDate" : ISODate("2013-01-16T23:15:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:15:12.427 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:15:14.428 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:15:16.007 [rsMgr] not electing self, we are not freshest STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:15:16.212 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:323639 161ms { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 295, "optime" : { "t" : 1358378116000, "i" : 1994 }, "optimeDate" : ISODate("2013-01-16T23:15:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 284, "optime" : { "t" : 1358378112000, "i" : 38 }, "optimeDate" : ISODate("2013-01-16T23:15:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 284, "optime" : { "t" : 1358378115000, "i" : 2927 }, "optimeDate" : ISODate("2013-01-16T23:15:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:15Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:16Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 294, "optime" : { "t" : 1358378115000, "i" : 3470 }, "optimeDate" : ISODate("2013-01-16T23:15:15Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 281, "optime" : { "t" : 1358378112000, "i" : 38 }, "optimeDate" : ISODate("2013-01-16T23:15:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 281, "optime" : { "t" : 1358378115000, "i" : 7093 }, "optimeDate" : ISODate("2013-01-16T23:15:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 281, "optime" : { "t" : 1358378112000, "i" : 38 }, "optimeDate" : ISODate("2013-01-16T23:15:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:15Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 293, "optime" : { "t" : 1358378116000, "i" : 1729 }, "optimeDate" : ISODate("2013-01-16T23:15:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:15:16.428 [conn18] end connection 127.0.0.2:53849 (2 connections now open) m31001| Wed Jan 16 18:15:16.428 [initandlisten] connection accepted from 127.0.0.2:40078 #20 (4 connections now open) m31000| Wed Jan 16 18:15:16.428 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:15:18.429 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:15:19.915 [conn20] end connection 127.0.0.4:39630 (4 connections now open) m31000| Wed Jan 16 18:15:19.916 [initandlisten] connection accepted from 127.0.0.4:38601 #21 (5 connections now open) m31000| Wed Jan 16 18:15:20.429 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 299, "optime" : { "t" : 1358378120000, "i" : 3865 }, "optimeDate" : ISODate("2013-01-16T23:15:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 288, "optime" : { "t" : 1358378119000, "i" : 2414 }, "optimeDate" : ISODate("2013-01-16T23:15:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 288, "optime" : { "t" : 1358378120000, "i" : 3487 }, "optimeDate" : ISODate("2013-01-16T23:15:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:20Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 298, "optime" : { "t" : 1358378119000, "i" : 2414 }, "optimeDate" : ISODate("2013-01-16T23:15:19Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 285, "optime" : { "t" : 1358378116000, "i" : 1994 }, "optimeDate" : ISODate("2013-01-16T23:15:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:19Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 285, "optime" : { "t" : 1358378119000, "i" : 5369 }, "optimeDate" : ISODate("2013-01-16T23:15:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 285, "optime" : { "t" : 1358378119000, "i" : 2414 }, "optimeDate" : ISODate("2013-01-16T23:15:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 297, "optime" : { "t" : 1358378120000, "i" : 3625 }, "optimeDate" : ISODate("2013-01-16T23:15:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:15:21.056 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31000| Wed Jan 16 18:15:22.430 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:15:24.430 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 303, "optime" : { "t" : 1358378124000, "i" : 5643 }, "optimeDate" : ISODate("2013-01-16T23:15:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 292, "optime" : { "t" : 1358378123000, "i" : 653 }, "optimeDate" : ISODate("2013-01-16T23:15:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 292, "optime" : { "t" : 1358378120000, "i" : 3865 }, "optimeDate" : ISODate("2013-01-16T23:15:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:23Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:24Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 302, "optime" : { "t" : 1358378123000, "i" : 653 }, "optimeDate" : ISODate("2013-01-16T23:15:23Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 289, "optime" : { "t" : 1358378120000, "i" : 3865 }, "optimeDate" : ISODate("2013-01-16T23:15:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 289, "optime" : { "t" : 1358378123000, "i" : 3390 }, "optimeDate" : ISODate("2013-01-16T23:15:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 289, "optime" : { "t" : 1358378120000, "i" : 3865 }, "optimeDate" : ISODate("2013-01-16T23:15:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 301, "optime" : { "t" : 1358378124000, "i" : 5349 }, "optimeDate" : ISODate("2013-01-16T23:15:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:15:26.430 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:15:27.057 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31002| Wed Jan 16 18:15:27.917 [conn20] end connection 127.0.0.2:54600 (4 connections now open) m31001| Wed Jan 16 18:15:28.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:15:28.417 [initandlisten] connection accepted from 127.0.0.2:32809 #22 (5 connections now open) m31000| Wed Jan 16 18:15:28.430 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 307, "optime" : { "t" : 1358378128000, "i" : 7661 }, "optimeDate" : ISODate("2013-01-16T23:15:28Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 296, "optime" : { "t" : 1358378124000, "i" : 5643 }, "optimeDate" : ISODate("2013-01-16T23:15:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 296, "optime" : { "t" : 1358378128000, "i" : 3426 }, "optimeDate" : ISODate("2013-01-16T23:15:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:28Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 306, "optime" : { "t" : 1358378128000, "i" : 2663 }, "optimeDate" : ISODate("2013-01-16T23:15:28Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 293, "optime" : { "t" : 1358378124000, "i" : 5643 }, "optimeDate" : ISODate("2013-01-16T23:15:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:27Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:28Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 293, "optime" : { "t" : 1358378127000, "i" : 1336 }, "optimeDate" : ISODate("2013-01-16T23:15:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 293, "optime" : { "t" : 1358378124000, "i" : 5643 }, "optimeDate" : ISODate("2013-01-16T23:15:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 305, "optime" : { "t" : 1358378128000, "i" : 6664 }, "optimeDate" : ISODate("2013-01-16T23:15:28Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:15:30.431 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:15:32.407 [conn19] end connection 127.0.0.4:54608 (2 connections now open) m31000| Wed Jan 16 18:15:32.431 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:15:32.907 [initandlisten] connection accepted from 127.0.0.4:57076 #21 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 311, "optime" : { "t" : 1358378132000, "i" : 8466 }, "optimeDate" : ISODate("2013-01-16T23:15:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 300, "optime" : { "t" : 1358378128000, "i" : 7661 }, "optimeDate" : ISODate("2013-01-16T23:15:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 300, "optime" : { "t" : 1358378128000, "i" : 7661 }, "optimeDate" : ISODate("2013-01-16T23:15:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:31Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:32Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 310, "optime" : { "t" : 1358378132000, "i" : 5600 }, "optimeDate" : ISODate("2013-01-16T23:15:32Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 297, "optime" : { "t" : 1358378128000, "i" : 7661 }, "optimeDate" : ISODate("2013-01-16T23:15:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:32Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 297, "optime" : { "t" : 1358378131000, "i" : 567 }, "optimeDate" : ISODate("2013-01-16T23:15:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 297, "optime" : { "t" : 1358378132000, "i" : 5600 }, "optimeDate" : ISODate("2013-01-16T23:15:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 309, "optime" : { "t" : 1358378132000, "i" : 7611 }, "optimeDate" : ISODate("2013-01-16T23:15:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:15:33.058 [conn21] end connection 127.0.0.3:39876 (4 connections now open) m31002| Wed Jan 16 18:15:33.058 [initandlisten] connection accepted from 127.0.0.3:44781 #23 (5 connections now open) m31001| Wed Jan 16 18:15:33.059 [rsMgr] not electing self, we are not freshest m31000| Wed Jan 16 18:15:34.431 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:15:36.432 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 316, "optime" : { "t" : 1358378137000, "i" : 825 }, "optimeDate" : ISODate("2013-01-16T23:15:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 305, "optime" : { "t" : 1358378132000, "i" : 8466 }, "optimeDate" : ISODate("2013-01-16T23:15:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 305, "optime" : { "t" : 1358378132000, "i" : 8466 }, "optimeDate" : ISODate("2013-01-16T23:15:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:35Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:37Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 315, "optime" : { "t" : 1358378136000, "i" : 6110 }, "optimeDate" : ISODate("2013-01-16T23:15:36Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 302, "optime" : { "t" : 1358378137000, "i" : 287 }, "optimeDate" : ISODate("2013-01-16T23:15:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 302, "optime" : { "t" : 1358378132000, "i" : 8466 }, "optimeDate" : ISODate("2013-01-16T23:15:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 302, "optime" : { "t" : 1358378132000, "i" : 8466 }, "optimeDate" : ISODate("2013-01-16T23:15:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 314, "optime" : { "t" : 1358378137000, "i" : 825 }, "optimeDate" : ISODate("2013-01-16T23:15:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:15:38.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:15:38.432 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:15:39.060 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31000| Wed Jan 16 18:15:40.432 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 320, "optime" : { "t" : 1358378141000, "i" : 2388 }, "optimeDate" : ISODate("2013-01-16T23:15:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 309, "optime" : { "t" : 1358378137000, "i" : 825 }, "optimeDate" : ISODate("2013-01-16T23:15:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 309, "optime" : { "t" : 1358378140000, "i" : 7298 }, "optimeDate" : ISODate("2013-01-16T23:15:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:41Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 319, "optime" : { "t" : 1358378140000, "i" : 5040 }, "optimeDate" : ISODate("2013-01-16T23:15:40Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 306, "optime" : { "t" : 1358378141000, "i" : 1056 }, "optimeDate" : ISODate("2013-01-16T23:15:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:41Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 306, "optime" : { "t" : 1358378137000, "i" : 825 }, "optimeDate" : ISODate("2013-01-16T23:15:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 306, "optime" : { "t" : 1358378137000, "i" : 825 }, "optimeDate" : ISODate("2013-01-16T23:15:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:41Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 318, "optime" : { "t" : 1358378141000, "i" : 2126 }, "optimeDate" : ISODate("2013-01-16T23:15:41Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:15:42.433 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:15:44.433 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:15:45.060 [rsMgr] not electing self, we are not freshest STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 324, "optime" : { "t" : 1358378145000, "i" : 4049 }, "optimeDate" : ISODate("2013-01-16T23:15:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 313, "optime" : { "t" : 1358378141000, "i" : 2388 }, "optimeDate" : ISODate("2013-01-16T23:15:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 313, "optime" : { "t" : 1358378141000, "i" : 2388 }, "optimeDate" : ISODate("2013-01-16T23:15:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:43Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:45Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 323, "optime" : { "t" : 1358378144000, "i" : 3642 }, "optimeDate" : ISODate("2013-01-16T23:15:44Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 310, "optime" : { "t" : 1358378145000, "i" : 821 }, "optimeDate" : ISODate("2013-01-16T23:15:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:45Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 310, "optime" : { "t" : 1358378141000, "i" : 2388 }, "optimeDate" : ISODate("2013-01-16T23:15:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 310, "optime" : { "t" : 1358378144000, "i" : 3642 }, "optimeDate" : ISODate("2013-01-16T23:15:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 322, "optime" : { "t" : 1358378145000, "i" : 4049 }, "optimeDate" : ISODate("2013-01-16T23:15:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:15:46.433 [conn20] end connection 127.0.0.2:40078 (2 connections now open) m31001| Wed Jan 16 18:15:46.433 [initandlisten] connection accepted from 127.0.0.2:58629 #22 (3 connections now open) m31000| Wed Jan 16 18:15:46.434 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:15:48.434 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 328, "optime" : { "t" : 1358378149000, "i" : 4705 }, "optimeDate" : ISODate("2013-01-16T23:15:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 317, "optime" : { "t" : 1358378145000, "i" : 4049 }, "optimeDate" : ISODate("2013-01-16T23:15:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 317, "optime" : { "t" : 1358378148000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:15:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:49Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 327, "optime" : { "t" : 1358378148000, "i" : 2853 }, "optimeDate" : ISODate("2013-01-16T23:15:48Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 314, "optime" : { "t" : 1358378149000, "i" : 400 }, "optimeDate" : ISODate("2013-01-16T23:15:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 314, "optime" : { "t" : 1358378145000, "i" : 4049 }, "optimeDate" : ISODate("2013-01-16T23:15:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 314, "optime" : { "t" : 1358378145000, "i" : 4049 }, "optimeDate" : ISODate("2013-01-16T23:15:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 326, "optime" : { "t" : 1358378149000, "i" : 4645 }, "optimeDate" : ISODate("2013-01-16T23:15:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:15:49.921 [conn21] end connection 127.0.0.4:38601 (4 connections now open) m31000| Wed Jan 16 18:15:49.921 [initandlisten] connection accepted from 127.0.0.4:54723 #22 (5 connections now open) m31000| Wed Jan 16 18:15:50.434 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:15:51.062 [rsMgr] not electing self, 127.0.0.4:31002 would veto with '127.0.0.3:31001 is trying to elect itself but 127.0.0.2:31000 is already primary and more up-to-date' m31000| Wed Jan 16 18:15:52.435 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 332, "optime" : { "t" : 1358378153000, "i" : 6450 }, "optimeDate" : ISODate("2013-01-16T23:15:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 321, "optime" : { "t" : 1358378149000, "i" : 4705 }, "optimeDate" : ISODate("2013-01-16T23:15:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 321, "optime" : { "t" : 1358378153000, "i" : 3809 }, "optimeDate" : ISODate("2013-01-16T23:15:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:53Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 331, "optime" : { "t" : 1358378152000, "i" : 1454 }, "optimeDate" : ISODate("2013-01-16T23:15:52Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 318, "optime" : { "t" : 1358378153000, "i" : 536 }, "optimeDate" : ISODate("2013-01-16T23:15:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 318, "optime" : { "t" : 1358378149000, "i" : 4705 }, "optimeDate" : ISODate("2013-01-16T23:15:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:53Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 318, "optime" : { "t" : 1358378152000, "i" : 1454 }, "optimeDate" : ISODate("2013-01-16T23:15:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:53Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 330, "optime" : { "t" : 1358378153000, "i" : 6403 }, "optimeDate" : ISODate("2013-01-16T23:15:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:15:54.435 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:15:56.435 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:15:57.062 [rsMgr] not electing self, we are not freshest STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 336, "optime" : { "t" : 1358378157000, "i" : 7577 }, "optimeDate" : ISODate("2013-01-16T23:15:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 325, "optime" : { "t" : 1358378153000, "i" : 6450 }, "optimeDate" : ISODate("2013-01-16T23:15:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 325, "optime" : { "t" : 1358378153000, "i" : 6450 }, "optimeDate" : ISODate("2013-01-16T23:15:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:55Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:57Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 335, "optime" : { "t" : 1358378156000, "i" : 309 }, "optimeDate" : ISODate("2013-01-16T23:15:56Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 322, "optime" : { "t" : 1358378157000, "i" : 377 }, "optimeDate" : ISODate("2013-01-16T23:15:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:15:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 322, "optime" : { "t" : 1358378153000, "i" : 6450 }, "optimeDate" : ISODate("2013-01-16T23:15:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 322, "optime" : { "t" : 1358378153000, "i" : 6450 }, "optimeDate" : ISODate("2013-01-16T23:15:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:15:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 334, "optime" : { "t" : 1358378157000, "i" : 7577 }, "optimeDate" : ISODate("2013-01-16T23:15:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:15:58.436 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:16:00.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:16:00.436 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:16:01.184 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:347036 173ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 340, "optime" : { "t" : 1358378161000, "i" : 8716 }, "optimeDate" : ISODate("2013-01-16T23:16:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 329, "optime" : { "t" : 1358378157000, "i" : 7577 }, "optimeDate" : ISODate("2013-01-16T23:15:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 329, "optime" : { "t" : 1358378160000, "i" : 551 }, "optimeDate" : ISODate("2013-01-16T23:16:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:01Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:01Z"), "myState" : 2, "syncingTo" : "127.0.0.4:31002", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 339, "optime" : { "t" : 1358378160000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:16:00Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 326, "optime" : { "t" : 1358378161000, "i" : 40 }, "optimeDate" : ISODate("2013-01-16T23:16:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 326, "optime" : { "t" : 1358378161000, "i" : 7977 }, "optimeDate" : ISODate("2013-01-16T23:16:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 326, "optime" : { "t" : 1358378157000, "i" : 7577 }, "optimeDate" : ISODate("2013-01-16T23:15:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 338, "optime" : { "t" : 1358378161000, "i" : 7977 }, "optimeDate" : ISODate("2013-01-16T23:16:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000" ], "graph" : { "127.0.0.4:31002" : [ "127.0.0.3:31001" ], "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:16:02.437 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:16:03.063 [conn23] end connection 127.0.0.3:44781 (4 connections now open) m31002| Wed Jan 16 18:16:03.063 [initandlisten] connection accepted from 127.0.0.3:35870 #24 (5 connections now open) m31001| Wed Jan 16 18:16:03.716 [rsMgr] replSet info electSelf 1 m31002| Wed Jan 16 18:16:03.716 [conn24] replSet info voting yea for 127.0.0.3:31001 (1) m31000| Wed Jan 16 18:16:04.437 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:16:05.862 [conn8] end connection 127.0.0.3:39284 (4 connections now open) m31002| Wed Jan 16 18:16:05.922 [conn22] end connection 127.0.0.2:32809 (3 connections now open) STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:16:06.269 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:477737 238ms m31001| Wed Jan 16 18:16:06.285 [rsMgr] replSet PRIMARY { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 345, "optime" : { "t" : 1358378166000, "i" : 1492 }, "optimeDate" : ISODate("2013-01-16T23:16:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 334, "optime" : { "t" : 1358378161000, "i" : 8716 }, "optimeDate" : ISODate("2013-01-16T23:16:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 334, "optime" : { "t" : 1358378161000, "i" : 8716 }, "optimeDate" : ISODate("2013-01-16T23:16:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:05Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 344, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 331, "optime" : { "t" : 1358378165000, "i" : 268 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 331, "optime" : { "t" : 1358378165000, "i" : 7342 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 331, "optime" : { "t" : 1358378161000, "i" : 8716 }, "optimeDate" : ISODate("2013-01-16T23:16:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 343, "optime" : { "t" : 1358378166000, "i" : 1492 }, "optimeDate" : ISODate("2013-01-16T23:16:06Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:16:06.422 [initandlisten] connection accepted from 127.0.0.2:51696 #25 (4 connections now open) m31000| Wed Jan 16 18:16:06.437 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:16:06.437 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state PRIMARY m31002| Wed Jan 16 18:16:07.912 [rsHealthPoll] replSet member 127.0.0.3:31001 is now in state PRIMARY m31002| Wed Jan 16 18:16:07.912 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:16:08.438 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:16:10.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 18:16:10.412 [conn21] end connection 127.0.0.4:57076 (2 connections now open) m31000| Wed Jan 16 18:16:10.438 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 349, "optime" : { "t" : 1358378170000, "i" : 3878 }, "optimeDate" : ISODate("2013-01-16T23:16:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 338, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 338, "optime" : { "t" : 1358378166000, "i" : 1492 }, "optimeDate" : ISODate("2013-01-16T23:16:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:09Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 348, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 335, "optime" : { "t" : 1358378166000, "i" : 1492 }, "optimeDate" : ISODate("2013-01-16T23:16:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 335, "optime" : { "t" : 1358378169000, "i" : 5402 }, "optimeDate" : ISODate("2013-01-16T23:16:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 335, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:09Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 347, "optime" : { "t" : 1358378170000, "i" : 3878 }, "optimeDate" : ISODate("2013-01-16T23:16:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:16:10.912 [initandlisten] connection accepted from 127.0.0.4:50643 #23 (3 connections now open) m31000| Wed Jan 16 18:16:12.438 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:16:13.413 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:16:14.439 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 353, "optime" : { "t" : 1358378174000, "i" : 5493 }, "optimeDate" : ISODate("2013-01-16T23:16:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 342, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 342, "optime" : { "t" : 1358378173000, "i" : 3198 }, "optimeDate" : ISODate("2013-01-16T23:16:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:13Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 352, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 339, "optime" : { "t" : 1358378170000, "i" : 3878 }, "optimeDate" : ISODate("2013-01-16T23:16:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:13Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 339, "optime" : { "t" : 1358378173000, "i" : 3516 }, "optimeDate" : ISODate("2013-01-16T23:16:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 339, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 351, "optime" : { "t" : 1358378174000, "i" : 5466 }, "optimeDate" : ISODate("2013-01-16T23:16:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:16:16.439 [conn22] end connection 127.0.0.2:58629 (2 connections now open) m31001| Wed Jan 16 18:16:16.439 [initandlisten] connection accepted from 127.0.0.2:47951 #24 (3 connections now open) m31000| Wed Jan 16 18:16:16.439 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:16:18.414 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:16:18.440 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 357, "optime" : { "t" : 1358378178000, "i" : 6162 }, "optimeDate" : ISODate("2013-01-16T23:16:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 346, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 346, "optime" : { "t" : 1358378174000, "i" : 5493 }, "optimeDate" : ISODate("2013-01-16T23:16:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:17Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:15:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 356, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 343, "optime" : { "t" : 1358378174000, "i" : 5493 }, "optimeDate" : ISODate("2013-01-16T23:16:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:18Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 343, "optime" : { "t" : 1358378177000, "i" : 2723 }, "optimeDate" : ISODate("2013-01-16T23:16:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 343, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 355, "optime" : { "t" : 1358378178000, "i" : 5900 }, "optimeDate" : ISODate("2013-01-16T23:16:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:16:19.926 [conn22] end connection 127.0.0.4:54723 (4 connections now open) m31000| Wed Jan 16 18:16:19.926 [initandlisten] connection accepted from 127.0.0.4:34777 #23 (5 connections now open) m31000| Wed Jan 16 18:16:20.440 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:16:22.440 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 361, "optime" : { "t" : 1358378182000, "i" : 7383 }, "optimeDate" : ISODate("2013-01-16T23:16:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 350, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 350, "optime" : { "t" : 1358378178000, "i" : 6162 }, "optimeDate" : ISODate("2013-01-16T23:16:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:21Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 360, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 347, "optime" : { "t" : 1358378178000, "i" : 6162 }, "optimeDate" : ISODate("2013-01-16T23:16:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 347, "optime" : { "t" : 1358378181000, "i" : 1464 }, "optimeDate" : ISODate("2013-01-16T23:16:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 347, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:21Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 359, "optime" : { "t" : 1358378182000, "i" : 7383 }, "optimeDate" : ISODate("2013-01-16T23:16:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:16:23.415 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:16:24.441 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:16:26.441 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 365, "optime" : { "t" : 1358378186000, "i" : 8536 }, "optimeDate" : ISODate("2013-01-16T23:16:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 354, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 354, "optime" : { "t" : 1358378186000, "i" : 3345 }, "optimeDate" : ISODate("2013-01-16T23:16:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 365, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 352, "optime" : { "t" : 1358378182000, "i" : 7383 }, "optimeDate" : ISODate("2013-01-16T23:16:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:25Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 352, "optime" : { "t" : 1358378185000, "i" : 764 }, "optimeDate" : ISODate("2013-01-16T23:16:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 352, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 364, "optime" : { "t" : 1358378186000, "i" : 8311 }, "optimeDate" : ISODate("2013-01-16T23:16:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:16:28.415 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:16:28.441 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:16:30.442 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 370, "optime" : { "t" : 1358378191000, "i" : 976 }, "optimeDate" : ISODate("2013-01-16T23:16:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 359, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 359, "optime" : { "t" : 1358378186000, "i" : 8536 }, "optimeDate" : ISODate("2013-01-16T23:16:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:29Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 369, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 356, "optime" : { "t" : 1358378191000, "i" : 652 }, "optimeDate" : ISODate("2013-01-16T23:16:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 356, "optime" : { "t" : 1358378186000, "i" : 8536 }, "optimeDate" : ISODate("2013-01-16T23:16:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 356, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:31Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 368, "optime" : { "t" : 1358378191000, "i" : 652 }, "optimeDate" : ISODate("2013-01-16T23:16:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:16:32.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:16:32.442 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:16:33.069 [conn24] end connection 127.0.0.3:35870 (3 connections now open) m31002| Wed Jan 16 18:16:33.069 [initandlisten] connection accepted from 127.0.0.3:51363 #26 (4 connections now open) m31002| Wed Jan 16 18:16:33.416 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:16:34.442 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 374, "optime" : { "t" : 1358378195000, "i" : 2700 }, "optimeDate" : ISODate("2013-01-16T23:16:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 363, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 363, "optime" : { "t" : 1358378191000, "i" : 976 }, "optimeDate" : ISODate("2013-01-16T23:16:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:33Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 373, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 360, "optime" : { "t" : 1358378195000, "i" : 929 }, "optimeDate" : ISODate("2013-01-16T23:16:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 360, "optime" : { "t" : 1358378191000, "i" : 976 }, "optimeDate" : ISODate("2013-01-16T23:16:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 360, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:35Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 372, "optime" : { "t" : 1358378195000, "i" : 2700 }, "optimeDate" : ISODate("2013-01-16T23:16:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:16:36.443 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:16:38.417 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:16:38.443 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 378, "optime" : { "t" : 1358378199000, "i" : 4527 }, "optimeDate" : ISODate("2013-01-16T23:16:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 367, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 367, "optime" : { "t" : 1358378198000, "i" : 4421 }, "optimeDate" : ISODate("2013-01-16T23:16:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 377, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 364, "optime" : { "t" : 1358378199000, "i" : 220 }, "optimeDate" : ISODate("2013-01-16T23:16:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 364, "optime" : { "t" : 1358378195000, "i" : 2700 }, "optimeDate" : ISODate("2013-01-16T23:16:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 364, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:39Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 376, "optime" : { "t" : 1358378199000, "i" : 4124 }, "optimeDate" : ISODate("2013-01-16T23:16:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:16:40.443 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:16:42.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:16:42.444 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:16:43.417 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 382, "optime" : { "t" : 1358378203000, "i" : 6041 }, "optimeDate" : ISODate("2013-01-16T23:16:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 371, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 371, "optime" : { "t" : 1358378199000, "i" : 4527 }, "optimeDate" : ISODate("2013-01-16T23:16:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:41Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 381, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 368, "optime" : { "t" : 1358378203000, "i" : 733 }, "optimeDate" : ISODate("2013-01-16T23:16:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:43Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 368, "optime" : { "t" : 1358378199000, "i" : 4527 }, "optimeDate" : ISODate("2013-01-16T23:16:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 368, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 380, "optime" : { "t" : 1358378203000, "i" : 5651 }, "optimeDate" : ISODate("2013-01-16T23:16:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:16:43.927 [conn25] end connection 127.0.0.2:51696 (3 connections now open) m31002| Wed Jan 16 18:16:44.427 [initandlisten] connection accepted from 127.0.0.2:39532 #27 (4 connections now open) m31000| Wed Jan 16 18:16:44.444 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:16:46.444 [conn24] end connection 127.0.0.2:47951 (2 connections now open) m31001| Wed Jan 16 18:16:46.445 [initandlisten] connection accepted from 127.0.0.2:60686 #25 (3 connections now open) m31000| Wed Jan 16 18:16:46.445 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 386, "optime" : { "t" : 1358378207000, "i" : 7173 }, "optimeDate" : ISODate("2013-01-16T23:16:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 375, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 375, "optime" : { "t" : 1358378206000, "i" : 1745 }, "optimeDate" : ISODate("2013-01-16T23:16:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 385, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 372, "optime" : { "t" : 1358378207000, "i" : 186 }, "optimeDate" : ISODate("2013-01-16T23:16:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 372, "optime" : { "t" : 1358378203000, "i" : 6041 }, "optimeDate" : ISODate("2013-01-16T23:16:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 372, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 384, "optime" : { "t" : 1358378207000, "i" : 6616 }, "optimeDate" : ISODate("2013-01-16T23:16:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:16:48.418 [conn23] end connection 127.0.0.4:50643 (2 connections now open) m31000| Wed Jan 16 18:16:48.445 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:16:48.918 [initandlisten] connection accepted from 127.0.0.4:55106 #26 (3 connections now open) m31002| Wed Jan 16 18:16:48.918 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:16:49.931 [conn23] end connection 127.0.0.4:34777 (4 connections now open) m31000| Wed Jan 16 18:16:49.931 [initandlisten] connection accepted from 127.0.0.4:56169 #24 (5 connections now open) m31000| Wed Jan 16 18:16:50.446 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 390, "optime" : { "t" : 1358378211000, "i" : 8382 }, "optimeDate" : ISODate("2013-01-16T23:16:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 379, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 379, "optime" : { "t" : 1358378207000, "i" : 7173 }, "optimeDate" : ISODate("2013-01-16T23:16:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:49Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 389, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 376, "optime" : { "t" : 1358378211000, "i" : 476 }, "optimeDate" : ISODate("2013-01-16T23:16:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:51Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 376, "optime" : { "t" : 1358378207000, "i" : 7173 }, "optimeDate" : ISODate("2013-01-16T23:16:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 376, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 388, "optime" : { "t" : 1358378211000, "i" : 8382 }, "optimeDate" : ISODate("2013-01-16T23:16:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:16:52.446 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:16:53.919 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:16:54.446 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 395, "optime" : { "t" : 1358378216000, "i" : 1071 }, "optimeDate" : ISODate("2013-01-16T23:16:56Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 384, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 384, "optime" : { "t" : 1358378211000, "i" : 8382 }, "optimeDate" : ISODate("2013-01-16T23:16:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:55Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 394, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 381, "optime" : { "t" : 1358378215000, "i" : 691 }, "optimeDate" : ISODate("2013-01-16T23:16:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:16:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 381, "optime" : { "t" : 1358378215000, "i" : 8349 }, "optimeDate" : ISODate("2013-01-16T23:16:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 381, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 393, "optime" : { "t" : 1358378216000, "i" : 1071 }, "optimeDate" : ISODate("2013-01-16T23:16:56Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:16:56.447 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:16:58.447 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:16:58.920 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 399, "optime" : { "t" : 1358378220000, "i" : 1750 }, "optimeDate" : ISODate("2013-01-16T23:17:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 388, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 388, "optime" : { "t" : 1358378219000, "i" : 3191 }, "optimeDate" : ISODate("2013-01-16T23:16:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:59Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 398, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 385, "optime" : { "t" : 1358378219000, "i" : 31 }, "optimeDate" : ISODate("2013-01-16T23:16:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 385, "optime" : { "t" : 1358378219000, "i" : 7236 }, "optimeDate" : ISODate("2013-01-16T23:16:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 385, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:16:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 397, "optime" : { "t" : 1358378220000, "i" : 1750 }, "optimeDate" : ISODate("2013-01-16T23:17:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:17:00.448 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:17:02.448 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:17:03.074 [conn26] end connection 127.0.0.3:51363 (3 connections now open) m31002| Wed Jan 16 18:17:03.074 [initandlisten] connection accepted from 127.0.0.3:48180 #28 (4 connections now open) m31002| Wed Jan 16 18:17:03.920 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:17:04.006 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:17:04.448 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 403, "optime" : { "t" : 1358378224000, "i" : 3543 }, "optimeDate" : ISODate("2013-01-16T23:17:04Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 392, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 392, "optime" : { "t" : 1358378224000, "i" : 3411 }, "optimeDate" : ISODate("2013-01-16T23:17:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 402, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 389, "optime" : { "t" : 1358378220000, "i" : 1750 }, "optimeDate" : ISODate("2013-01-16T23:17:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:03Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 389, "optime" : { "t" : 1358378223000, "i" : 5605 }, "optimeDate" : ISODate("2013-01-16T23:17:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 389, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 401, "optime" : { "t" : 1358378224000, "i" : 3411 }, "optimeDate" : ISODate("2013-01-16T23:17:04Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:17:06.449 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:17:08.449 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 407, "optime" : { "t" : 1358378228000, "i" : 4828 }, "optimeDate" : ISODate("2013-01-16T23:17:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 396, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 396, "optime" : { "t" : 1358378224000, "i" : 3543 }, "optimeDate" : ISODate("2013-01-16T23:17:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:07Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 406, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 393, "optime" : { "t" : 1358378224000, "i" : 3543 }, "optimeDate" : ISODate("2013-01-16T23:17:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 393, "optime" : { "t" : 1358378227000, "i" : 4192 }, "optimeDate" : ISODate("2013-01-16T23:17:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 393, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:07Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 405, "optime" : { "t" : 1358378228000, "i" : 4828 }, "optimeDate" : ISODate("2013-01-16T23:17:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:17:08.921 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:17:10.449 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:17:12.450 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 411, "optime" : { "t" : 1358378232000, "i" : 6202 }, "optimeDate" : ISODate("2013-01-16T23:17:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 400, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 400, "optime" : { "t" : 1358378231000, "i" : 2623 }, "optimeDate" : ISODate("2013-01-16T23:17:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:11Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 410, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 397, "optime" : { "t" : 1358378228000, "i" : 4828 }, "optimeDate" : ISODate("2013-01-16T23:17:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:11Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 397, "optime" : { "t" : 1358378231000, "i" : 3058 }, "optimeDate" : ISODate("2013-01-16T23:17:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 397, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 409, "optime" : { "t" : 1358378232000, "i" : 6202 }, "optimeDate" : ISODate("2013-01-16T23:17:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:17:13.922 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:17:14.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:17:14.450 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:17:16.285 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:293530 146ms m31001| Wed Jan 16 18:17:16.450 [conn25] end connection 127.0.0.2:60686 (2 connections now open) m31001| Wed Jan 16 18:17:16.450 [initandlisten] connection accepted from 127.0.0.2:52511 #27 (4 connections now open) m31000| Wed Jan 16 18:17:16.451 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 415, "optime" : { "t" : 1358378236000, "i" : 7985 }, "optimeDate" : ISODate("2013-01-16T23:17:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 404, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 404, "optime" : { "t" : 1358378232000, "i" : 6202 }, "optimeDate" : ISODate("2013-01-16T23:17:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:15Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 414, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 401, "optime" : { "t" : 1358378232000, "i" : 6202 }, "optimeDate" : ISODate("2013-01-16T23:17:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:16Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 401, "optime" : { "t" : 1358378235000, "i" : 1780 }, "optimeDate" : ISODate("2013-01-16T23:17:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 401, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 413, "optime" : { "t" : 1358378236000, "i" : 7985 }, "optimeDate" : ISODate("2013-01-16T23:17:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:17:18.451 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:17:18.923 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:17:19.936 [conn24] end connection 127.0.0.4:56169 (4 connections now open) m31000| Wed Jan 16 18:17:19.936 [initandlisten] connection accepted from 127.0.0.4:44260 #25 (6 connections now open) m31000| Wed Jan 16 18:17:20.451 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 420, "optime" : { "t" : 1358378241000, "i" : 1319 }, "optimeDate" : ISODate("2013-01-16T23:17:21Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 409, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 409, "optime" : { "t" : 1358378236000, "i" : 7985 }, "optimeDate" : ISODate("2013-01-16T23:17:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:19Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:16:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 419, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 406, "optime" : { "t" : 1358378241000, "i" : 973 }, "optimeDate" : ISODate("2013-01-16T23:17:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 406, "optime" : { "t" : 1358378239000, "i" : 10 }, "optimeDate" : ISODate("2013-01-16T23:17:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 406, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:21Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 418, "optime" : { "t" : 1358378241000, "i" : 1319 }, "optimeDate" : ISODate("2013-01-16T23:17:21Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:17:21.933 [conn27] end connection 127.0.0.2:39532 (3 connections now open) m31002| Wed Jan 16 18:17:22.433 [initandlisten] connection accepted from 127.0.0.2:42330 #29 (4 connections now open) m31000| Wed Jan 16 18:17:22.452 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:17:23.924 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:17:24.452 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 424, "optime" : { "t" : 1358378245000, "i" : 3767 }, "optimeDate" : ISODate("2013-01-16T23:17:25Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 413, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 413, "optime" : { "t" : 1358378244000, "i" : 5763 }, "optimeDate" : ISODate("2013-01-16T23:17:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 423, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 410, "optime" : { "t" : 1358378245000, "i" : 778 }, "optimeDate" : ISODate("2013-01-16T23:17:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:25Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 410, "optime" : { "t" : 1358378241000, "i" : 1319 }, "optimeDate" : ISODate("2013-01-16T23:17:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 410, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 422, "optime" : { "t" : 1358378245000, "i" : 3767 }, "optimeDate" : ISODate("2013-01-16T23:17:25Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:17:26.424 [conn26] end connection 127.0.0.4:55106 (2 connections now open) m31000| Wed Jan 16 18:17:26.452 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:17:26.924 [initandlisten] connection accepted from 127.0.0.4:34893 #28 (3 connections now open) m31000| Wed Jan 16 18:17:28.453 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:17:29.424 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 428, "optime" : { "t" : 1358378249000, "i" : 5635 }, "optimeDate" : ISODate("2013-01-16T23:17:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 417, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 417, "optime" : { "t" : 1358378245000, "i" : 3767 }, "optimeDate" : ISODate("2013-01-16T23:17:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:27Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 427, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 414, "optime" : { "t" : 1358378249000, "i" : 389 }, "optimeDate" : ISODate("2013-01-16T23:17:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:29Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 414, "optime" : { "t" : 1358378245000, "i" : 3767 }, "optimeDate" : ISODate("2013-01-16T23:17:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 414, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 426, "optime" : { "t" : 1358378249000, "i" : 5635 }, "optimeDate" : ISODate("2013-01-16T23:17:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:17:30.453 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:17:32.454 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:17:33.080 [conn28] end connection 127.0.0.3:48180 (3 connections now open) m31002| Wed Jan 16 18:17:33.080 [initandlisten] connection accepted from 127.0.0.3:34736 #30 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 432, "optime" : { "t" : 1358378253000, "i" : 7469 }, "optimeDate" : ISODate("2013-01-16T23:17:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 421, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 421, "optime" : { "t" : 1358378249000, "i" : 5635 }, "optimeDate" : ISODate("2013-01-16T23:17:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 431, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 418, "optime" : { "t" : 1358378253000, "i" : 560 }, "optimeDate" : ISODate("2013-01-16T23:17:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 418, "optime" : { "t" : 1358378249000, "i" : 5635 }, "optimeDate" : ISODate("2013-01-16T23:17:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 418, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:33Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 430, "optime" : { "t" : 1358378253000, "i" : 7469 }, "optimeDate" : ISODate("2013-01-16T23:17:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:17:34.425 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:17:34.454 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:17:36.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:17:36.454 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 437, "optime" : { "t" : 1358378258000, "i" : 1046 }, "optimeDate" : ISODate("2013-01-16T23:17:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 426, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 426, "optime" : { "t" : 1358378257000, "i" : 3336 }, "optimeDate" : ISODate("2013-01-16T23:17:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:37Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 436, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 423, "optime" : { "t" : 1358378257000, "i" : 485 }, "optimeDate" : ISODate("2013-01-16T23:17:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 423, "optime" : { "t" : 1358378257000, "i" : 7450 }, "optimeDate" : ISODate("2013-01-16T23:17:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 423, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 435, "optime" : { "t" : 1358378258000, "i" : 1046 }, "optimeDate" : ISODate("2013-01-16T23:17:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:17:38.455 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:17:39.426 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:17:40.455 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:17:41.372 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:322372 161ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 441, "optime" : { "t" : 1358378262000, "i" : 2403 }, "optimeDate" : ISODate("2013-01-16T23:17:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 430, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 430, "optime" : { "t" : 1358378258000, "i" : 1046 }, "optimeDate" : ISODate("2013-01-16T23:17:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:41Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 440, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 427, "optime" : { "t" : 1358378258000, "i" : 1046 }, "optimeDate" : ISODate("2013-01-16T23:17:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:41Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 427, "optime" : { "t" : 1358378261000, "i" : 7400 }, "optimeDate" : ISODate("2013-01-16T23:17:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 427, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 439, "optime" : { "t" : 1358378262000, "i" : 2292 }, "optimeDate" : ISODate("2013-01-16T23:17:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:17:42.455 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:17:44.427 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:17:44.456 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:17:46.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 445, "optime" : { "t" : 1358378266000, "i" : 4131 }, "optimeDate" : ISODate("2013-01-16T23:17:46Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 434, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 434, "optime" : { "t" : 1358378262000, "i" : 2403 }, "optimeDate" : ISODate("2013-01-16T23:17:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:45Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 444, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 431, "optime" : { "t" : 1358378262000, "i" : 2403 }, "optimeDate" : ISODate("2013-01-16T23:17:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 431, "optime" : { "t" : 1358378265000, "i" : 5718 }, "optimeDate" : ISODate("2013-01-16T23:17:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 431, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 443, "optime" : { "t" : 1358378266000, "i" : 4089 }, "optimeDate" : ISODate("2013-01-16T23:17:46Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:17:46.456 [conn27] end connection 127.0.0.2:52511 (2 connections now open) m31001| Wed Jan 16 18:17:46.456 [initandlisten] connection accepted from 127.0.0.2:55761 #29 (4 connections now open) m31000| Wed Jan 16 18:17:46.456 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:17:48.457 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:17:49.427 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:17:49.941 [conn25] end connection 127.0.0.4:44260 (4 connections now open) m31000| Wed Jan 16 18:17:49.942 [initandlisten] connection accepted from 127.0.0.4:42214 #26 (5 connections now open) m31000| Wed Jan 16 18:17:50.457 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 449, "optime" : { "t" : 1358378270000, "i" : 5488 }, "optimeDate" : ISODate("2013-01-16T23:17:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 438, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 438, "optime" : { "t" : 1358378269000, "i" : 4196 }, "optimeDate" : ISODate("2013-01-16T23:17:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:49Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 448, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 435, "optime" : { "t" : 1358378266000, "i" : 4131 }, "optimeDate" : ISODate("2013-01-16T23:17:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:49Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 435, "optime" : { "t" : 1358378269000, "i" : 4250 }, "optimeDate" : ISODate("2013-01-16T23:17:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 435, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 447, "optime" : { "t" : 1358378270000, "i" : 5488 }, "optimeDate" : ISODate("2013-01-16T23:17:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:17:52.458 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:17:54.428 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:17:54.458 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 453, "optime" : { "t" : 1358378274000, "i" : 6714 }, "optimeDate" : ISODate("2013-01-16T23:17:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 442, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 442, "optime" : { "t" : 1358378270000, "i" : 5488 }, "optimeDate" : ISODate("2013-01-16T23:17:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:53Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 452, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 439, "optime" : { "t" : 1358378270000, "i" : 5488 }, "optimeDate" : ISODate("2013-01-16T23:17:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:54Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 439, "optime" : { "t" : 1358378273000, "i" : 3032 }, "optimeDate" : ISODate("2013-01-16T23:17:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 439, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 451, "optime" : { "t" : 1358378274000, "i" : 6557 }, "optimeDate" : ISODate("2013-01-16T23:17:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:17:56.458 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:17:58.458 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 457, "optime" : { "t" : 1358378278000, "i" : 7993 }, "optimeDate" : ISODate("2013-01-16T23:17:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 446, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 446, "optime" : { "t" : 1358378274000, "i" : 6714 }, "optimeDate" : ISODate("2013-01-16T23:17:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:57Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 456, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 443, "optime" : { "t" : 1358378274000, "i" : 6714 }, "optimeDate" : ISODate("2013-01-16T23:17:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:17:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 443, "optime" : { "t" : 1358378277000, "i" : 1531 }, "optimeDate" : ISODate("2013-01-16T23:17:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 443, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:17:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 455, "optime" : { "t" : 1358378278000, "i" : 7951 }, "optimeDate" : ISODate("2013-01-16T23:17:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:17:59.429 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:17:59.938 [conn29] end connection 127.0.0.2:42330 (3 connections now open) m31002| Wed Jan 16 18:18:00.438 [initandlisten] connection accepted from 127.0.0.2:50095 #31 (4 connections now open) m31000| Wed Jan 16 18:18:00.459 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:18:02.459 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 462, "optime" : { "t" : 1358378283000, "i" : 677 }, "optimeDate" : ISODate("2013-01-16T23:18:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 451, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 451, "optime" : { "t" : 1358378282000, "i" : 8311 }, "optimeDate" : ISODate("2013-01-16T23:18:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 461, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 448, "optime" : { "t" : 1358378278000, "i" : 7993 }, "optimeDate" : ISODate("2013-01-16T23:17:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:01Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 448, "optime" : { "t" : 1358378281000, "i" : 93 }, "optimeDate" : ISODate("2013-01-16T23:18:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 448, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 460, "optime" : { "t" : 1358378283000, "i" : 538 }, "optimeDate" : ISODate("2013-01-16T23:18:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:18:03.085 [conn30] end connection 127.0.0.3:34736 (3 connections now open) m31002| Wed Jan 16 18:18:03.085 [initandlisten] connection accepted from 127.0.0.3:37240 #32 (4 connections now open) m31001| Wed Jan 16 18:18:04.429 [conn28] end connection 127.0.0.4:34893 (2 connections now open) m31000| Wed Jan 16 18:18:04.459 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:18:04.929 [initandlisten] connection accepted from 127.0.0.4:56404 #30 (3 connections now open) m31002| Wed Jan 16 18:18:04.930 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:18:06.460 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 466, "optime" : { "t" : 1358378287000, "i" : 2581 }, "optimeDate" : ISODate("2013-01-16T23:18:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 455, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 455, "optime" : { "t" : 1358378283000, "i" : 677 }, "optimeDate" : ISODate("2013-01-16T23:18:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:05Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 465, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 452, "optime" : { "t" : 1358378287000, "i" : 750 }, "optimeDate" : ISODate("2013-01-16T23:18:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 452, "optime" : { "t" : 1358378283000, "i" : 677 }, "optimeDate" : ISODate("2013-01-16T23:18:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 452, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:07Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 464, "optime" : { "t" : 1358378287000, "i" : 2581 }, "optimeDate" : ISODate("2013-01-16T23:18:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:18:08.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:18:08.460 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:18:09.930 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:18:10.460 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 470, "optime" : { "t" : 1358378291000, "i" : 3744 }, "optimeDate" : ISODate("2013-01-16T23:18:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 459, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 459, "optime" : { "t" : 1358378290000, "i" : 1203 }, "optimeDate" : ISODate("2013-01-16T23:18:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 469, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 456, "optime" : { "t" : 1358378291000, "i" : 390 }, "optimeDate" : ISODate("2013-01-16T23:18:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 456, "optime" : { "t" : 1358378287000, "i" : 2581 }, "optimeDate" : ISODate("2013-01-16T23:18:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 456, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 468, "optime" : { "t" : 1358378291000, "i" : 3673 }, "optimeDate" : ISODate("2013-01-16T23:18:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:18:12.461 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:18:14.461 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:18:14.931 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 474, "optime" : { "t" : 1358378295000, "i" : 4612 }, "optimeDate" : ISODate("2013-01-16T23:18:15Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 463, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 463, "optime" : { "t" : 1358378295000, "i" : 3821 }, "optimeDate" : ISODate("2013-01-16T23:18:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 473, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 460, "optime" : { "t" : 1358378295000, "i" : 888 }, "optimeDate" : ISODate("2013-01-16T23:18:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 460, "optime" : { "t" : 1358378291000, "i" : 3744 }, "optimeDate" : ISODate("2013-01-16T23:18:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:15Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 460, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:15Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 472, "optime" : { "t" : 1358378295000, "i" : 4612 }, "optimeDate" : ISODate("2013-01-16T23:18:15Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:18:16.461 [conn29] end connection 127.0.0.2:55761 (2 connections now open) m31001| Wed Jan 16 18:18:16.461 [initandlisten] connection accepted from 127.0.0.2:42053 #31 (3 connections now open) m31000| Wed Jan 16 18:18:16.462 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:18:18.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:18:18.462 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 478, "optime" : { "t" : 1358378299000, "i" : 6086 }, "optimeDate" : ISODate("2013-01-16T23:18:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 467, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 467, "optime" : { "t" : 1358378295000, "i" : 4612 }, "optimeDate" : ISODate("2013-01-16T23:18:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:17Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 477, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 464, "optime" : { "t" : 1358378299000, "i" : 433 }, "optimeDate" : ISODate("2013-01-16T23:18:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 464, "optime" : { "t" : 1358378295000, "i" : 4612 }, "optimeDate" : ISODate("2013-01-16T23:18:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 464, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:19Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 476, "optime" : { "t" : 1358378299000, "i" : 6086 }, "optimeDate" : ISODate("2013-01-16T23:18:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:18:19.932 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:18:19.947 [conn26] end connection 127.0.0.4:42214 (4 connections now open) m31000| Wed Jan 16 18:18:19.947 [initandlisten] connection accepted from 127.0.0.4:53965 #27 (5 connections now open) m31000| Wed Jan 16 18:18:20.463 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:18:22.463 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:18:23.656 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:243655 121ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 482, "optime" : { "t" : 1358378303000, "i" : 7239 }, "optimeDate" : ISODate("2013-01-16T23:18:23Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 471, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 471, "optime" : { "t" : 1358378302000, "i" : 1715 }, "optimeDate" : ISODate("2013-01-16T23:18:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 481, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 468, "optime" : { "t" : 1358378303000, "i" : 193 }, "optimeDate" : ISODate("2013-01-16T23:18:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 468, "optime" : { "t" : 1358378299000, "i" : 6086 }, "optimeDate" : ISODate("2013-01-16T23:18:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 468, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 480, "optime" : { "t" : 1358378303000, "i" : 7239 }, "optimeDate" : ISODate("2013-01-16T23:18:23Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:18:24.463 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:18:24.932 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:18:26.464 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 486, "optime" : { "t" : 1358378307000, "i" : 8434 }, "optimeDate" : ISODate("2013-01-16T23:18:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 475, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 475, "optime" : { "t" : 1358378307000, "i" : 8020 }, "optimeDate" : ISODate("2013-01-16T23:18:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:27Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:17:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 485, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 472, "optime" : { "t" : 1358378307000, "i" : 111 }, "optimeDate" : ISODate("2013-01-16T23:18:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:27Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 472, "optime" : { "t" : 1358378307000, "i" : 8209 }, "optimeDate" : ISODate("2013-01-16T23:18:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 472, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 484, "optime" : { "t" : 1358378307000, "i" : 8201 }, "optimeDate" : ISODate("2013-01-16T23:18:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:18:28.464 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:18:29.933 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:18:30.465 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:18:31.239 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:241907 120ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 491, "optime" : { "t" : 1358378312000, "i" : 353 }, "optimeDate" : ISODate("2013-01-16T23:18:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 480, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 480, "optime" : { "t" : 1358378307000, "i" : 8434 }, "optimeDate" : ISODate("2013-01-16T23:18:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:31Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 490, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 477, "optime" : { "t" : 1358378311000, "i" : 1048 }, "optimeDate" : ISODate("2013-01-16T23:18:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 477, "optime" : { "t" : 1358378311000, "i" : 9157 }, "optimeDate" : ISODate("2013-01-16T23:18:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 477, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:31Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 489, "optime" : { "t" : 1358378312000, "i" : 4 }, "optimeDate" : ISODate("2013-01-16T23:18:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:18:32.465 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:18:33.090 [conn32] end connection 127.0.0.3:37240 (3 connections now open) m31002| Wed Jan 16 18:18:33.090 [initandlisten] connection accepted from 127.0.0.3:35076 #33 (4 connections now open) m31000| Wed Jan 16 18:18:34.465 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:18:34.934 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 495, "optime" : { "t" : 1358378316000, "i" : 2151 }, "optimeDate" : ISODate("2013-01-16T23:18:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 484, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 484, "optime" : { "t" : 1358378315000, "i" : 2445 }, "optimeDate" : ISODate("2013-01-16T23:18:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:35Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 494, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 481, "optime" : { "t" : 1358378312000, "i" : 353 }, "optimeDate" : ISODate("2013-01-16T23:18:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 481, "optime" : { "t" : 1358378315000, "i" : 7425 }, "optimeDate" : ISODate("2013-01-16T23:18:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 481, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:35Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 493, "optime" : { "t" : 1358378316000, "i" : 1817 }, "optimeDate" : ISODate("2013-01-16T23:18:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:18:36.466 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:18:37.943 [conn31] end connection 127.0.0.2:50095 (3 connections now open) m31002| Wed Jan 16 18:18:38.444 [initandlisten] connection accepted from 127.0.0.2:47683 #34 (4 connections now open) m31000| Wed Jan 16 18:18:38.466 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:18:39.934 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:18:40.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:18:40.467 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 499, "optime" : { "t" : 1358378320000, "i" : 4214 }, "optimeDate" : ISODate("2013-01-16T23:18:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 488, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 488, "optime" : { "t" : 1358378316000, "i" : 2151 }, "optimeDate" : ISODate("2013-01-16T23:18:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:39Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 498, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 485, "optime" : { "t" : 1358378316000, "i" : 2151 }, "optimeDate" : ISODate("2013-01-16T23:18:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:39Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 485, "optime" : { "t" : 1358378319000, "i" : 5550 }, "optimeDate" : ISODate("2013-01-16T23:18:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 485, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 497, "optime" : { "t" : 1358378320000, "i" : 4214 }, "optimeDate" : ISODate("2013-01-16T23:18:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:18:42.435 [conn30] end connection 127.0.0.4:56404 (2 connections now open) m31000| Wed Jan 16 18:18:42.467 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:18:42.935 [initandlisten] connection accepted from 127.0.0.4:43182 #32 (3 connections now open) m31000| Wed Jan 16 18:18:44.467 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 503, "optime" : { "t" : 1358378324000, "i" : 5596 }, "optimeDate" : ISODate("2013-01-16T23:18:44Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 492, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 492, "optime" : { "t" : 1358378320000, "i" : 4214 }, "optimeDate" : ISODate("2013-01-16T23:18:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:43Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 502, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 489, "optime" : { "t" : 1358378320000, "i" : 4214 }, "optimeDate" : ISODate("2013-01-16T23:18:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 489, "optime" : { "t" : 1358378323000, "i" : 4156 }, "optimeDate" : ISODate("2013-01-16T23:18:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 489, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:43Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 501, "optime" : { "t" : 1358378324000, "i" : 5318 }, "optimeDate" : ISODate("2013-01-16T23:18:44Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:18:45.436 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:18:46.468 [conn31] end connection 127.0.0.2:42053 (2 connections now open) m31001| Wed Jan 16 18:18:46.468 [initandlisten] connection accepted from 127.0.0.2:46592 #33 (3 connections now open) m31000| Wed Jan 16 18:18:46.468 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:18:48.468 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 507, "optime" : { "t" : 1358378328000, "i" : 6966 }, "optimeDate" : ISODate("2013-01-16T23:18:48Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 496, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 496, "optime" : { "t" : 1358378328000, "i" : 3918 }, "optimeDate" : ISODate("2013-01-16T23:18:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 506, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 493, "optime" : { "t" : 1358378324000, "i" : 5596 }, "optimeDate" : ISODate("2013-01-16T23:18:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:47Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 493, "optime" : { "t" : 1358378327000, "i" : 2717 }, "optimeDate" : ISODate("2013-01-16T23:18:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 493, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 505, "optime" : { "t" : 1358378328000, "i" : 6966 }, "optimeDate" : ISODate("2013-01-16T23:18:48Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:18:49.951 [conn27] end connection 127.0.0.4:53965 (4 connections now open) m31000| Wed Jan 16 18:18:49.952 [initandlisten] connection accepted from 127.0.0.4:44439 #28 (5 connections now open) m31001| Wed Jan 16 18:18:50.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:18:50.436 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:18:50.469 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:18:52.469 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 512, "optime" : { "t" : 1358378333000, "i" : 447 }, "optimeDate" : ISODate("2013-01-16T23:18:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 501, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 501, "optime" : { "t" : 1358378328000, "i" : 6966 }, "optimeDate" : ISODate("2013-01-16T23:18:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:51Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 511, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 498, "optime" : { "t" : 1358378328000, "i" : 6966 }, "optimeDate" : ISODate("2013-01-16T23:18:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:52Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 498, "optime" : { "t" : 1358378331000, "i" : 292 }, "optimeDate" : ISODate("2013-01-16T23:18:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 498, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:53Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 510, "optime" : { "t" : 1358378333000, "i" : 447 }, "optimeDate" : ISODate("2013-01-16T23:18:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:18:54.469 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:18:55.437 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:18:56.470 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 516, "optime" : { "t" : 1358378337000, "i" : 2461 }, "optimeDate" : ISODate("2013-01-16T23:18:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 505, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 505, "optime" : { "t" : 1358378333000, "i" : 447 }, "optimeDate" : ISODate("2013-01-16T23:18:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:55Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 515, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 502, "optime" : { "t" : 1358378337000, "i" : 874 }, "optimeDate" : ISODate("2013-01-16T23:18:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:18:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 502, "optime" : { "t" : 1358378333000, "i" : 447 }, "optimeDate" : ISODate("2013-01-16T23:18:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 502, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:18:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 514, "optime" : { "t" : 1358378337000, "i" : 2461 }, "optimeDate" : ISODate("2013-01-16T23:18:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:18:58.470 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:19:00.437 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:19:00.470 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:19:01.329 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:223964 111ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 520, "optime" : { "t" : 1358378341000, "i" : 3900 }, "optimeDate" : ISODate("2013-01-16T23:19:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 509, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 509, "optime" : { "t" : 1358378340000, "i" : 5461 }, "optimeDate" : ISODate("2013-01-16T23:19:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 519, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 506, "optime" : { "t" : 1358378341000, "i" : 512 }, "optimeDate" : ISODate("2013-01-16T23:19:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 506, "optime" : { "t" : 1358378337000, "i" : 2461 }, "optimeDate" : ISODate("2013-01-16T23:18:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 506, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:18:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 518, "optime" : { "t" : 1358378341000, "i" : 3900 }, "optimeDate" : ISODate("2013-01-16T23:19:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:19:02.471 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:19:03.096 [conn33] end connection 127.0.0.3:35076 (3 connections now open) m31002| Wed Jan 16 18:19:03.096 [initandlisten] connection accepted from 127.0.0.3:38627 #35 (4 connections now open) m31000| Wed Jan 16 18:19:04.471 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:19:05.438 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 524, "optime" : { "t" : 1358378345000, "i" : 4746 }, "optimeDate" : ISODate("2013-01-16T23:19:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 513, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 513, "optime" : { "t" : 1358378341000, "i" : 3900 }, "optimeDate" : ISODate("2013-01-16T23:19:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:03Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 523, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 510, "optime" : { "t" : 1358378345000, "i" : 54 }, "optimeDate" : ISODate("2013-01-16T23:19:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:05Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 510, "optime" : { "t" : 1358378341000, "i" : 3900 }, "optimeDate" : ISODate("2013-01-16T23:19:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 510, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 522, "optime" : { "t" : 1358378345000, "i" : 4281 }, "optimeDate" : ISODate("2013-01-16T23:19:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:19:06.471 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:19:08.472 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 528, "optime" : { "t" : 1358378349000, "i" : 5748 }, "optimeDate" : ISODate("2013-01-16T23:19:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 517, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 517, "optime" : { "t" : 1358378345000, "i" : 4746 }, "optimeDate" : ISODate("2013-01-16T23:19:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 527, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 514, "optime" : { "t" : 1358378349000, "i" : 927 }, "optimeDate" : ISODate("2013-01-16T23:19:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 514, "optime" : { "t" : 1358378345000, "i" : 4746 }, "optimeDate" : ISODate("2013-01-16T23:19:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 514, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:09Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 526, "optime" : { "t" : 1358378349000, "i" : 5511 }, "optimeDate" : ISODate("2013-01-16T23:19:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:19:10.439 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:19:10.472 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:19:12.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:19:12.473 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 532, "optime" : { "t" : 1358378353000, "i" : 7989 }, "optimeDate" : ISODate("2013-01-16T23:19:13Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 521, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 521, "optime" : { "t" : 1358378353000, "i" : 3836 }, "optimeDate" : ISODate("2013-01-16T23:19:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 531, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 518, "optime" : { "t" : 1358378353000, "i" : 949 }, "optimeDate" : ISODate("2013-01-16T23:19:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 518, "optime" : { "t" : 1358378349000, "i" : 5748 }, "optimeDate" : ISODate("2013-01-16T23:19:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 518, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:13Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 530, "optime" : { "t" : 1358378353000, "i" : 7989 }, "optimeDate" : ISODate("2013-01-16T23:19:13Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:19:14.473 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:19:15.440 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:19:15.949 [conn34] end connection 127.0.0.2:47683 (3 connections now open) m31002| Wed Jan 16 18:19:16.449 [initandlisten] connection accepted from 127.0.0.2:48152 #36 (4 connections now open) m31001| Wed Jan 16 18:19:16.473 [conn33] end connection 127.0.0.2:46592 (2 connections now open) m31001| Wed Jan 16 18:19:16.473 [initandlisten] connection accepted from 127.0.0.2:46690 #34 (3 connections now open) m31000| Wed Jan 16 18:19:16.474 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 537, "optime" : { "t" : 1358378358000, "i" : 751 }, "optimeDate" : ISODate("2013-01-16T23:19:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 526, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 526, "optime" : { "t" : 1358378353000, "i" : 7989 }, "optimeDate" : ISODate("2013-01-16T23:19:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:17Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 536, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 523, "optime" : { "t" : 1358378357000, "i" : 1063 }, "optimeDate" : ISODate("2013-01-16T23:19:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:17Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 523, "optime" : { "t" : 1358378357000, "i" : 8100 }, "optimeDate" : ISODate("2013-01-16T23:19:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 523, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 535, "optime" : { "t" : 1358378358000, "i" : 751 }, "optimeDate" : ISODate("2013-01-16T23:19:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:19:18.474 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:19:19.957 [conn28] end connection 127.0.0.4:44439 (4 connections now open) m31000| Wed Jan 16 18:19:19.957 [initandlisten] connection accepted from 127.0.0.4:38716 #29 (5 connections now open) m31001| Wed Jan 16 18:19:20.440 [conn32] end connection 127.0.0.4:43182 (2 connections now open) m31000| Wed Jan 16 18:19:20.474 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:19:20.940 [initandlisten] connection accepted from 127.0.0.4:58882 #35 (3 connections now open) m31002| Wed Jan 16 18:19:20.941 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:19:21.279 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:294230 147ms m31000| Wed Jan 16 18:19:21.365 [FileAllocator] allocating new datafile /data/db/testReplSet-0/foo.3, filling with zeroes... m31001| Wed Jan 16 18:19:22.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:19:22.387 [FileAllocator] done allocating datafile /data/db/testReplSet-0/foo.3, size: 128MB, took 1.021 secs m31000| Wed Jan 16 18:19:22.388 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:1022375 1022ms m31002| Wed Jan 16 18:19:22.388 [FileAllocator] allocating new datafile /data/db/testReplSet-2/foo.3, filling with zeroes... m31000| Wed Jan 16 18:19:22.475 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 541, "optime" : { "t" : 1358378362000, "i" : 7792 }, "optimeDate" : ISODate("2013-01-16T23:19:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 530, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 530, "optime" : { "t" : 1358378361000, "i" : 2208 }, "optimeDate" : ISODate("2013-01-16T23:19:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:21Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 540, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 527, "optime" : { "t" : 1358378361000, "i" : 61 }, "optimeDate" : ISODate("2013-01-16T23:19:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 527, "optime" : { "t" : 1358378361000, "i" : 2208 }, "optimeDate" : ISODate("2013-01-16T23:19:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 527, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:21Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 539, "optime" : { "t" : 1358378361000, "i" : 2208 }, "optimeDate" : ISODate("2013-01-16T23:19:21Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:19:23.422 [FileAllocator] done allocating datafile /data/db/testReplSet-2/foo.3, size: 128MB, took 1.033 secs m31000| Wed Jan 16 18:19:24.475 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:19:25.941 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:19:26.475 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 545, "optime" : { "t" : 1358378366000, "i" : 8075 }, "optimeDate" : ISODate("2013-01-16T23:19:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 534, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 534, "optime" : { "t" : 1358378366000, "i" : 4511 }, "optimeDate" : ISODate("2013-01-16T23:19:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 544, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 531, "optime" : { "t" : 1358378362000, "i" : 7792 }, "optimeDate" : ISODate("2013-01-16T23:19:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:25Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 531, "optime" : { "t" : 1358378365000, "i" : 1453 }, "optimeDate" : ISODate("2013-01-16T23:19:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 531, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 543, "optime" : { "t" : 1358378366000, "i" : 7665 }, "optimeDate" : ISODate("2013-01-16T23:19:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:19:28.476 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:19:30.476 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:19:30.942 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 550, "optime" : { "t" : 1358378371000, "i" : 316 }, "optimeDate" : ISODate("2013-01-16T23:19:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 539, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 539, "optime" : { "t" : 1358378366000, "i" : 8075 }, "optimeDate" : ISODate("2013-01-16T23:19:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:29Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 549, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 536, "optime" : { "t" : 1358378366000, "i" : 8075 }, "optimeDate" : ISODate("2013-01-16T23:19:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:30Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 536, "optime" : { "t" : 1358378369000, "i" : 521 }, "optimeDate" : ISODate("2013-01-16T23:19:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 536, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 548, "optime" : { "t" : 1358378371000, "i" : 316 }, "optimeDate" : ISODate("2013-01-16T23:19:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:19:32.476 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:19:33.101 [conn35] end connection 127.0.0.3:38627 (3 connections now open) m31002| Wed Jan 16 18:19:33.102 [initandlisten] connection accepted from 127.0.0.3:55501 #37 (4 connections now open) m31000| Wed Jan 16 18:19:34.477 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 554, "optime" : { "t" : 1358378375000, "i" : 1874 }, "optimeDate" : ISODate("2013-01-16T23:19:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 543, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 543, "optime" : { "t" : 1358378371000, "i" : 316 }, "optimeDate" : ISODate("2013-01-16T23:19:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:33Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 553, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 540, "optime" : { "t" : 1358378375000, "i" : 771 }, "optimeDate" : ISODate("2013-01-16T23:19:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 540, "optime" : { "t" : 1358378371000, "i" : 316 }, "optimeDate" : ISODate("2013-01-16T23:19:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 540, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:35Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 552, "optime" : { "t" : 1358378375000, "i" : 1681 }, "optimeDate" : ISODate("2013-01-16T23:19:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:19:35.943 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:19:36.477 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:19:38.477 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 558, "optime" : { "t" : 1358378379000, "i" : 2530 }, "optimeDate" : ISODate("2013-01-16T23:19:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 547, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 547, "optime" : { "t" : 1358378378000, "i" : 6545 }, "optimeDate" : ISODate("2013-01-16T23:19:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 557, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 544, "optime" : { "t" : 1358378379000, "i" : 529 }, "optimeDate" : ISODate("2013-01-16T23:19:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 544, "optime" : { "t" : 1358378375000, "i" : 1874 }, "optimeDate" : ISODate("2013-01-16T23:19:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 544, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:39Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 556, "optime" : { "t" : 1358378379000, "i" : 2027 }, "optimeDate" : ISODate("2013-01-16T23:19:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:19:40.478 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:19:40.943 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:19:42.478 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 562, "optime" : { "t" : 1358378383000, "i" : 4034 }, "optimeDate" : ISODate("2013-01-16T23:19:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 551, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 551, "optime" : { "t" : 1358378379000, "i" : 2530 }, "optimeDate" : ISODate("2013-01-16T23:19:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:41Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 561, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 548, "optime" : { "t" : 1358378383000, "i" : 1084 }, "optimeDate" : ISODate("2013-01-16T23:19:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:43Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 548, "optime" : { "t" : 1358378379000, "i" : 2530 }, "optimeDate" : ISODate("2013-01-16T23:19:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 548, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 560, "optime" : { "t" : 1358378383000, "i" : 3862 }, "optimeDate" : ISODate("2013-01-16T23:19:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:19:44.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:19:44.478 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:19:45.944 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:19:46.479 [conn34] end connection 127.0.0.2:46690 (2 connections now open) m31001| Wed Jan 16 18:19:46.479 [initandlisten] connection accepted from 127.0.0.2:48677 #36 (4 connections now open) m31000| Wed Jan 16 18:19:46.479 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 566, "optime" : { "t" : 1358378387000, "i" : 5644 }, "optimeDate" : ISODate("2013-01-16T23:19:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 555, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 555, "optime" : { "t" : 1358378383000, "i" : 4034 }, "optimeDate" : ISODate("2013-01-16T23:19:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 565, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 552, "optime" : { "t" : 1358378387000, "i" : 350 }, "optimeDate" : ISODate("2013-01-16T23:19:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 552, "optime" : { "t" : 1358378383000, "i" : 4034 }, "optimeDate" : ISODate("2013-01-16T23:19:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 552, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 564, "optime" : { "t" : 1358378387000, "i" : 5526 }, "optimeDate" : ISODate("2013-01-16T23:19:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:19:48.479 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:19:49.963 [conn29] end connection 127.0.0.4:38716 (4 connections now open) m31000| Wed Jan 16 18:19:49.963 [initandlisten] connection accepted from 127.0.0.4:34245 #30 (5 connections now open) m31000| Wed Jan 16 18:19:50.480 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:19:50.945 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 570, "optime" : { "t" : 1358378391000, "i" : 6841 }, "optimeDate" : ISODate("2013-01-16T23:19:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 559, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 559, "optime" : { "t" : 1358378391000, "i" : 3482 }, "optimeDate" : ISODate("2013-01-16T23:19:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 569, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 556, "optime" : { "t" : 1358378391000, "i" : 488 }, "optimeDate" : ISODate("2013-01-16T23:19:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 556, "optime" : { "t" : 1358378387000, "i" : 5644 }, "optimeDate" : ISODate("2013-01-16T23:19:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:51Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 556, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 568, "optime" : { "t" : 1358378391000, "i" : 6841 }, "optimeDate" : ISODate("2013-01-16T23:19:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:19:52.480 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:19:53.954 [conn36] end connection 127.0.0.2:48152 (3 connections now open) m31001| Wed Jan 16 18:19:54.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:19:54.455 [initandlisten] connection accepted from 127.0.0.2:37178 #38 (4 connections now open) m31000| Wed Jan 16 18:19:54.481 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 m31002| Wed Jan 16 18:19:55.945 [rsMgr] replSet info two primaries (transiently) { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 574, "optime" : { "t" : 1358378395000, "i" : 8437 }, "optimeDate" : ISODate("2013-01-16T23:19:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 563, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 563, "optime" : { "t" : 1358378391000, "i" : 6841 }, "optimeDate" : ISODate("2013-01-16T23:19:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:55Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 573, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 560, "optime" : { "t" : 1358378395000, "i" : 558 }, "optimeDate" : ISODate("2013-01-16T23:19:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:55Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:19:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 560, "optime" : { "t" : 1358378395000, "i" : 8017 }, "optimeDate" : ISODate("2013-01-16T23:19:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 560, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 572, "optime" : { "t" : 1358378395000, "i" : 8437 }, "optimeDate" : ISODate("2013-01-16T23:19:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:19:56.481 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:19:58.445 [conn35] end connection 127.0.0.4:58882 (2 connections now open) m31000| Wed Jan 16 18:19:58.481 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:19:58.945 [initandlisten] connection accepted from 127.0.0.4:47357 #37 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 579, "optime" : { "t" : 1358378400000, "i" : 128 }, "optimeDate" : ISODate("2013-01-16T23:20:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 568, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 568, "optime" : { "t" : 1358378399000, "i" : 4441 }, "optimeDate" : ISODate("2013-01-16T23:19:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:59Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 578, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 565, "optime" : { "t" : 1358378399000, "i" : 713 }, "optimeDate" : ISODate("2013-01-16T23:19:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 565, "optime" : { "t" : 1358378399000, "i" : 9509 }, "optimeDate" : ISODate("2013-01-16T23:19:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 565, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:19:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:19:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 577, "optime" : { "t" : 1358378399000, "i" : 9772 }, "optimeDate" : ISODate("2013-01-16T23:19:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:20:00.482 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:20:01.446 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:20:02.482 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:20:03.106 [conn37] end connection 127.0.0.3:55501 (3 connections now open) m31002| Wed Jan 16 18:20:03.106 [initandlisten] connection accepted from 127.0.0.3:52554 #39 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 583, "optime" : { "t" : 1358378404000, "i" : 1863 }, "optimeDate" : ISODate("2013-01-16T23:20:04Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 572, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 572, "optime" : { "t" : 1358378400000, "i" : 128 }, "optimeDate" : ISODate("2013-01-16T23:20:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:03Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 582, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 569, "optime" : { "t" : 1358378403000, "i" : 123 }, "optimeDate" : ISODate("2013-01-16T23:20:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:03Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 569, "optime" : { "t" : 1358378403000, "i" : 7666 }, "optimeDate" : ISODate("2013-01-16T23:20:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 569, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 581, "optime" : { "t" : 1358378404000, "i" : 1641 }, "optimeDate" : ISODate("2013-01-16T23:20:04Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:20:04.483 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:20:06.447 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:20:06.483 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 587, "optime" : { "t" : 1358378408000, "i" : 3597 }, "optimeDate" : ISODate("2013-01-16T23:20:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 576, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 576, "optime" : { "t" : 1358378404000, "i" : 1863 }, "optimeDate" : ISODate("2013-01-16T23:20:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:07Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 586, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 573, "optime" : { "t" : 1358378404000, "i" : 1863 }, "optimeDate" : ISODate("2013-01-16T23:20:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 573, "optime" : { "t" : 1358378407000, "i" : 6038 }, "optimeDate" : ISODate("2013-01-16T23:20:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 573, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:07Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 585, "optime" : { "t" : 1358378408000, "i" : 2936 }, "optimeDate" : ISODate("2013-01-16T23:20:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:20:08.483 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:20:10.484 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:20:11.448 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:20:12.453 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:261423 130ms m31000| Wed Jan 16 18:20:12.484 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 591, "optime" : { "t" : 1358378412000, "i" : 4077 }, "optimeDate" : ISODate("2013-01-16T23:20:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 580, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 580, "optime" : { "t" : 1358378411000, "i" : 5626 }, "optimeDate" : ISODate("2013-01-16T23:20:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:11Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 590, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 577, "optime" : { "t" : 1358378408000, "i" : 3597 }, "optimeDate" : ISODate("2013-01-16T23:20:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:11Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 577, "optime" : { "t" : 1358378411000, "i" : 5701 }, "optimeDate" : ISODate("2013-01-16T23:20:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 577, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 589, "optime" : { "t" : 1358378412000, "i" : 4072 }, "optimeDate" : ISODate("2013-01-16T23:20:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:20:14.484 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:20:16.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:20:16.448 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:20:16.484 [conn36] end connection 127.0.0.2:48677 (2 connections now open) m31001| Wed Jan 16 18:20:16.485 [initandlisten] connection accepted from 127.0.0.2:51487 #38 (3 connections now open) m31000| Wed Jan 16 18:20:16.485 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 595, "optime" : { "t" : 1358378416000, "i" : 5695 }, "optimeDate" : ISODate("2013-01-16T23:20:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 584, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 584, "optime" : { "t" : 1358378412000, "i" : 4077 }, "optimeDate" : ISODate("2013-01-16T23:20:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:15Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 594, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 581, "optime" : { "t" : 1358378412000, "i" : 4077 }, "optimeDate" : ISODate("2013-01-16T23:20:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:16Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 581, "optime" : { "t" : 1358378415000, "i" : 4100 }, "optimeDate" : ISODate("2013-01-16T23:20:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 581, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 593, "optime" : { "t" : 1358378416000, "i" : 5146 }, "optimeDate" : ISODate("2013-01-16T23:20:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:20:18.485 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:20:19.968 [conn30] end connection 127.0.0.4:34245 (4 connections now open) m31000| Wed Jan 16 18:20:19.968 [initandlisten] connection accepted from 127.0.0.4:50595 #31 (5 connections now open) m31000| Wed Jan 16 18:20:20.485 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 599, "optime" : { "t" : 1358378420000, "i" : 6988 }, "optimeDate" : ISODate("2013-01-16T23:20:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 588, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 588, "optime" : { "t" : 1358378416000, "i" : 5695 }, "optimeDate" : ISODate("2013-01-16T23:20:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:19Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 598, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 585, "optime" : { "t" : 1358378416000, "i" : 5695 }, "optimeDate" : ISODate("2013-01-16T23:20:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 585, "optime" : { "t" : 1358378419000, "i" : 3011 }, "optimeDate" : ISODate("2013-01-16T23:20:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 585, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:19Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 597, "optime" : { "t" : 1358378420000, "i" : 6988 }, "optimeDate" : ISODate("2013-01-16T23:20:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:20:21.449 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:20:22.486 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:20:24.486 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 603, "optime" : { "t" : 1358378424000, "i" : 7589 }, "optimeDate" : ISODate("2013-01-16T23:20:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 592, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 592, "optime" : { "t" : 1358378424000, "i" : 4144 }, "optimeDate" : ISODate("2013-01-16T23:20:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 602, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 589, "optime" : { "t" : 1358378420000, "i" : 6988 }, "optimeDate" : ISODate("2013-01-16T23:20:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:23Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 589, "optime" : { "t" : 1358378423000, "i" : 2128 }, "optimeDate" : ISODate("2013-01-16T23:20:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 589, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 601, "optime" : { "t" : 1358378424000, "i" : 7408 }, "optimeDate" : ISODate("2013-01-16T23:20:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:20:26.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:20:26.449 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:20:26.487 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:20:28.487 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 608, "optime" : { "t" : 1358378429000, "i" : 221 }, "optimeDate" : ISODate("2013-01-16T23:20:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 597, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 597, "optime" : { "t" : 1358378424000, "i" : 7589 }, "optimeDate" : ISODate("2013-01-16T23:20:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:27Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 607, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 594, "optime" : { "t" : 1358378424000, "i" : 7589 }, "optimeDate" : ISODate("2013-01-16T23:20:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:28Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 594, "optime" : { "t" : 1358378427000, "i" : 751 }, "optimeDate" : ISODate("2013-01-16T23:20:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 594, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 606, "optime" : { "t" : 1358378428000, "i" : 8792 }, "optimeDate" : ISODate("2013-01-16T23:20:28Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:20:30.487 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:20:31.450 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:20:31.960 [conn38] end connection 127.0.0.2:37178 (3 connections now open) m31000| Wed Jan 16 18:20:32.439 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:420219 210ms m31002| Wed Jan 16 18:20:32.460 [initandlisten] connection accepted from 127.0.0.2:34351 #40 (4 connections now open) m31000| Wed Jan 16 18:20:32.487 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 612, "optime" : { "t" : 1358378433000, "i" : 471 }, "optimeDate" : ISODate("2013-01-16T23:20:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 601, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 601, "optime" : { "t" : 1358378432000, "i" : 1841 }, "optimeDate" : ISODate("2013-01-16T23:20:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 611, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 598, "optime" : { "t" : 1358378429000, "i" : 221 }, "optimeDate" : ISODate("2013-01-16T23:20:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:31Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 598, "optime" : { "t" : 1358378429000, "i" : 221 }, "optimeDate" : ISODate("2013-01-16T23:20:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 598, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 610, "optime" : { "t" : 1358378433000, "i" : 8 }, "optimeDate" : ISODate("2013-01-16T23:20:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:20:33.112 [conn39] end connection 127.0.0.3:52554 (3 connections now open) m31002| Wed Jan 16 18:20:33.112 [initandlisten] connection accepted from 127.0.0.3:52707 #41 (4 connections now open) m31000| Wed Jan 16 18:20:34.488 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:20:36.451 [conn37] end connection 127.0.0.4:47357 (2 connections now open) m31000| Wed Jan 16 18:20:36.488 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:20:36.951 [initandlisten] connection accepted from 127.0.0.4:45260 #39 (3 connections now open) m31002| Wed Jan 16 18:20:36.951 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 616, "optime" : { "t" : 1358378437000, "i" : 1900 }, "optimeDate" : ISODate("2013-01-16T23:20:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 605, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 605, "optime" : { "t" : 1358378433000, "i" : 471 }, "optimeDate" : ISODate("2013-01-16T23:20:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:35Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 615, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 602, "optime" : { "t" : 1358378437000, "i" : 1192 }, "optimeDate" : ISODate("2013-01-16T23:20:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 602, "optime" : { "t" : 1358378433000, "i" : 471 }, "optimeDate" : ISODate("2013-01-16T23:20:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 602, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 614, "optime" : { "t" : 1358378437000, "i" : 1288 }, "optimeDate" : ISODate("2013-01-16T23:20:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:20:38.488 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:20:40.489 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 620, "optime" : { "t" : 1358378441000, "i" : 2544 }, "optimeDate" : ISODate("2013-01-16T23:20:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 609, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 609, "optime" : { "t" : 1358378437000, "i" : 1900 }, "optimeDate" : ISODate("2013-01-16T23:20:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:39Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 619, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 606, "optime" : { "t" : 1358378441000, "i" : 322 }, "optimeDate" : ISODate("2013-01-16T23:20:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:41Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 606, "optime" : { "t" : 1358378437000, "i" : 1900 }, "optimeDate" : ISODate("2013-01-16T23:20:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 606, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:41Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 618, "optime" : { "t" : 1358378441000, "i" : 1883 }, "optimeDate" : ISODate("2013-01-16T23:20:41Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:20:41.952 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:20:42.489 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:20:44.489 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 624, "optime" : { "t" : 1358378445000, "i" : 4020 }, "optimeDate" : ISODate("2013-01-16T23:20:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 613, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 613, "optime" : { "t" : 1358378444000, "i" : 5842 }, "optimeDate" : ISODate("2013-01-16T23:20:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 623, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 610, "optime" : { "t" : 1358378445000, "i" : 1086 }, "optimeDate" : ISODate("2013-01-16T23:20:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 610, "optime" : { "t" : 1358378441000, "i" : 2544 }, "optimeDate" : ISODate("2013-01-16T23:20:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 610, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 622, "optime" : { "t" : 1358378445000, "i" : 4020 }, "optimeDate" : ISODate("2013-01-16T23:20:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:20:46.490 [conn38] end connection 127.0.0.2:51487 (2 connections now open) m31001| Wed Jan 16 18:20:46.490 [initandlisten] connection accepted from 127.0.0.2:41505 #40 (3 connections now open) m31000| Wed Jan 16 18:20:46.490 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:20:46.952 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:20:48.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:20:48.491 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 628, "optime" : { "t" : 1358378449000, "i" : 6035 }, "optimeDate" : ISODate("2013-01-16T23:20:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 617, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 617, "optime" : { "t" : 1358378445000, "i" : 4020 }, "optimeDate" : ISODate("2013-01-16T23:20:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:47Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 627, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 614, "optime" : { "t" : 1358378449000, "i" : 1064 }, "optimeDate" : ISODate("2013-01-16T23:20:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:49Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 614, "optime" : { "t" : 1358378445000, "i" : 4020 }, "optimeDate" : ISODate("2013-01-16T23:20:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 614, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 626, "optime" : { "t" : 1358378449000, "i" : 6035 }, "optimeDate" : ISODate("2013-01-16T23:20:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:20:49.974 [conn31] end connection 127.0.0.4:50595 (4 connections now open) m31000| Wed Jan 16 18:20:49.974 [initandlisten] connection accepted from 127.0.0.4:48593 #32 (5 connections now open) m31000| Wed Jan 16 18:20:50.491 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:20:51.953 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:20:52.491 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 632, "optime" : { "t" : 1358378453000, "i" : 7233 }, "optimeDate" : ISODate("2013-01-16T23:20:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 621, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 621, "optime" : { "t" : 1358378449000, "i" : 6035 }, "optimeDate" : ISODate("2013-01-16T23:20:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 631, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 618, "optime" : { "t" : 1358378453000, "i" : 473 }, "optimeDate" : ISODate("2013-01-16T23:20:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 618, "optime" : { "t" : 1358378449000, "i" : 6035 }, "optimeDate" : ISODate("2013-01-16T23:20:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 618, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:53Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 630, "optime" : { "t" : 1358378453000, "i" : 6386 }, "optimeDate" : ISODate("2013-01-16T23:20:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:20:54.492 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:20:56.492 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:20:56.954 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 636, "optime" : { "t" : 1358378457000, "i" : 8327 }, "optimeDate" : ISODate("2013-01-16T23:20:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 625, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 625, "optime" : { "t" : 1358378457000, "i" : 3996 }, "optimeDate" : ISODate("2013-01-16T23:20:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:57Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 635, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 622, "optime" : { "t" : 1358378457000, "i" : 847 }, "optimeDate" : ISODate("2013-01-16T23:20:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:20:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 622, "optime" : { "t" : 1358378457000, "i" : 8077 }, "optimeDate" : ISODate("2013-01-16T23:20:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 622, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:20:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 634, "optime" : { "t" : 1358378457000, "i" : 8327 }, "optimeDate" : ISODate("2013-01-16T23:20:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:20:58.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:20:58.493 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:21:00.493 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:21:01.954 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 641, "optime" : { "t" : 1358378462000, "i" : 1179 }, "optimeDate" : ISODate("2013-01-16T23:21:02Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 630, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 630, "optime" : { "t" : 1358378457000, "i" : 8327 }, "optimeDate" : ISODate("2013-01-16T23:20:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:01Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 640, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 627, "optime" : { "t" : 1358378461000, "i" : 410 }, "optimeDate" : ISODate("2013-01-16T23:21:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:01Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 627, "optime" : { "t" : 1358378461000, "i" : 8569 }, "optimeDate" : ISODate("2013-01-16T23:21:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 627, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 639, "optime" : { "t" : 1358378462000, "i" : 446 }, "optimeDate" : ISODate("2013-01-16T23:21:02Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:21:02.493 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:21:03.117 [conn41] end connection 127.0.0.3:52707 (3 connections now open) m31002| Wed Jan 16 18:21:03.118 [initandlisten] connection accepted from 127.0.0.3:60311 #42 (4 connections now open) m31000| Wed Jan 16 18:21:04.494 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 645, "optime" : { "t" : 1358378466000, "i" : 2109 }, "optimeDate" : ISODate("2013-01-16T23:21:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 634, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 634, "optime" : { "t" : 1358378462000, "i" : 1179 }, "optimeDate" : ISODate("2013-01-16T23:21:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:05Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:20:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 644, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 631, "optime" : { "t" : 1358378462000, "i" : 1179 }, "optimeDate" : ISODate("2013-01-16T23:21:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 631, "optime" : { "t" : 1358378465000, "i" : 7864 }, "optimeDate" : ISODate("2013-01-16T23:21:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 631, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:05Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 643, "optime" : { "t" : 1358378466000, "i" : 2109 }, "optimeDate" : ISODate("2013-01-16T23:21:06Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:21:06.494 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:21:06.955 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:21:08.495 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:21:09.965 [conn40] end connection 127.0.0.2:34351 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 649, "optime" : { "t" : 1358378470000, "i" : 2778 }, "optimeDate" : ISODate("2013-01-16T23:21:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 638, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 638, "optime" : { "t" : 1358378466000, "i" : 2109 }, "optimeDate" : ISODate("2013-01-16T23:21:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:09Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 648, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 635, "optime" : { "t" : 1358378466000, "i" : 2109 }, "optimeDate" : ISODate("2013-01-16T23:21:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:09Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 635, "optime" : { "t" : 1358378469000, "i" : 6975 }, "optimeDate" : ISODate("2013-01-16T23:21:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 635, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 647, "optime" : { "t" : 1358378470000, "i" : 2778 }, "optimeDate" : ISODate("2013-01-16T23:21:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:21:10.465 [initandlisten] connection accepted from 127.0.0.2:43087 #43 (4 connections now open) m31000| Wed Jan 16 18:21:10.495 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:21:11.956 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:21:12.495 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 653, "optime" : { "t" : 1358378474000, "i" : 3458 }, "optimeDate" : ISODate("2013-01-16T23:21:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 642, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 642, "optime" : { "t" : 1358378470000, "i" : 2778 }, "optimeDate" : ISODate("2013-01-16T23:21:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:13Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 652, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 639, "optime" : { "t" : 1358378470000, "i" : 2778 }, "optimeDate" : ISODate("2013-01-16T23:21:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 639, "optime" : { "t" : 1358378473000, "i" : 6144 }, "optimeDate" : ISODate("2013-01-16T23:21:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 639, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:13Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 651, "optime" : { "t" : 1358378474000, "i" : 3271 }, "optimeDate" : ISODate("2013-01-16T23:21:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:21:14.456 [conn39] end connection 127.0.0.4:45260 (2 connections now open) m31000| Wed Jan 16 18:21:14.496 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:21:14.956 [initandlisten] connection accepted from 127.0.0.4:45716 #41 (3 connections now open) m31001| Wed Jan 16 18:21:16.496 [conn40] end connection 127.0.0.2:41505 (2 connections now open) m31001| Wed Jan 16 18:21:16.496 [initandlisten] connection accepted from 127.0.0.2:33406 #42 (3 connections now open) m31000| Wed Jan 16 18:21:16.497 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:21:17.457 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:21:18.497 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 657, "optime" : { "t" : 1358378478000, "i" : 4871 }, "optimeDate" : ISODate("2013-01-16T23:21:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 646, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 646, "optime" : { "t" : 1358378477000, "i" : 4420 }, "optimeDate" : ISODate("2013-01-16T23:21:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:17Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 656, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 643, "optime" : { "t" : 1358378474000, "i" : 3458 }, "optimeDate" : ISODate("2013-01-16T23:21:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:17Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 643, "optime" : { "t" : 1358378477000, "i" : 4955 }, "optimeDate" : ISODate("2013-01-16T23:21:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 643, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 655, "optime" : { "t" : 1358378478000, "i" : 3980 }, "optimeDate" : ISODate("2013-01-16T23:21:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:21:19.979 [conn32] end connection 127.0.0.4:48593 (4 connections now open) m31000| Wed Jan 16 18:21:19.979 [initandlisten] connection accepted from 127.0.0.4:46780 #33 (5 connections now open) m31001| Wed Jan 16 18:21:20.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:21:20.497 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:21:22.285 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:275946 137ms m31002| Wed Jan 16 18:21:22.457 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:21:22.498 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 661, "optime" : { "t" : 1358378482000, "i" : 6301 }, "optimeDate" : ISODate("2013-01-16T23:21:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 650, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 650, "optime" : { "t" : 1358378478000, "i" : 4871 }, "optimeDate" : ISODate("2013-01-16T23:21:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:21Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 660, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 647, "optime" : { "t" : 1358378478000, "i" : 4871 }, "optimeDate" : ISODate("2013-01-16T23:21:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:22Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 647, "optime" : { "t" : 1358378481000, "i" : 3698 }, "optimeDate" : ISODate("2013-01-16T23:21:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 647, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 659, "optime" : { "t" : 1358378482000, "i" : 6301 }, "optimeDate" : ISODate("2013-01-16T23:21:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:21:24.498 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:21:26.498 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 665, "optime" : { "t" : 1358378486000, "i" : 7416 }, "optimeDate" : ISODate("2013-01-16T23:21:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 654, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 654, "optime" : { "t" : 1358378482000, "i" : 6301 }, "optimeDate" : ISODate("2013-01-16T23:21:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:25Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 664, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 651, "optime" : { "t" : 1358378482000, "i" : 6301 }, "optimeDate" : ISODate("2013-01-16T23:21:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 651, "optime" : { "t" : 1358378485000, "i" : 2419 }, "optimeDate" : ISODate("2013-01-16T23:21:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 651, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 663, "optime" : { "t" : 1358378486000, "i" : 7416 }, "optimeDate" : ISODate("2013-01-16T23:21:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:21:27.458 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:21:28.499 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:21:30.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:21:30.499 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 670, "optime" : { "t" : 1358378491000, "i" : 803 }, "optimeDate" : ISODate("2013-01-16T23:21:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 659, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 659, "optime" : { "t" : 1358378490000, "i" : 3002 }, "optimeDate" : ISODate("2013-01-16T23:21:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 669, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 656, "optime" : { "t" : 1358378486000, "i" : 7416 }, "optimeDate" : ISODate("2013-01-16T23:21:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:29Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 656, "optime" : { "t" : 1358378489000, "i" : 479 }, "optimeDate" : ISODate("2013-01-16T23:21:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 656, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 668, "optime" : { "t" : 1358378491000, "i" : 81 }, "optimeDate" : ISODate("2013-01-16T23:21:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:21:32.459 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:21:32.499 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:21:33.124 [conn42] end connection 127.0.0.3:60311 (3 connections now open) m31002| Wed Jan 16 18:21:33.124 [initandlisten] connection accepted from 127.0.0.3:58374 #44 (4 connections now open) m31000| Wed Jan 16 18:21:34.500 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 674, "optime" : { "t" : 1358378495000, "i" : 1662 }, "optimeDate" : ISODate("2013-01-16T23:21:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 663, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 663, "optime" : { "t" : 1358378491000, "i" : 803 }, "optimeDate" : ISODate("2013-01-16T23:21:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:33Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 673, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 660, "optime" : { "t" : 1358378495000, "i" : 968 }, "optimeDate" : ISODate("2013-01-16T23:21:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 660, "optime" : { "t" : 1358378491000, "i" : 803 }, "optimeDate" : ISODate("2013-01-16T23:21:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 660, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:35Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 672, "optime" : { "t" : 1358378495000, "i" : 1662 }, "optimeDate" : ISODate("2013-01-16T23:21:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:21:36.500 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:21:37.460 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:21:38.500 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:21:38.534 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:210732 105ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 678, "optime" : { "t" : 1358378499000, "i" : 2566 }, "optimeDate" : ISODate("2013-01-16T23:21:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 667, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 667, "optime" : { "t" : 1358378495000, "i" : 1662 }, "optimeDate" : ISODate("2013-01-16T23:21:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:37Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 677, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 664, "optime" : { "t" : 1358378499000, "i" : 727 }, "optimeDate" : ISODate("2013-01-16T23:21:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 664, "optime" : { "t" : 1358378495000, "i" : 1662 }, "optimeDate" : ISODate("2013-01-16T23:21:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 664, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:39Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 676, "optime" : { "t" : 1358378499000, "i" : 2566 }, "optimeDate" : ISODate("2013-01-16T23:21:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:21:40.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:21:40.501 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:21:42.460 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:21:42.501 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:21:42.575 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:575798 287ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 682, "optime" : { "t" : 1358378503000, "i" : 2466 }, "optimeDate" : ISODate("2013-01-16T23:21:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 671, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 671, "optime" : { "t" : 1358378502000, "i" : 7534 }, "optimeDate" : ISODate("2013-01-16T23:21:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 681, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 668, "optime" : { "t" : 1358378503000, "i" : 1499 }, "optimeDate" : ISODate("2013-01-16T23:21:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 668, "optime" : { "t" : 1358378499000, "i" : 2566 }, "optimeDate" : ISODate("2013-01-16T23:21:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 668, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:43Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 680, "optime" : { "t" : 1358378503000, "i" : 2213 }, "optimeDate" : ISODate("2013-01-16T23:21:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:21:44.502 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:21:46.502 [conn42] end connection 127.0.0.2:33406 (2 connections now open) m31001| Wed Jan 16 18:21:46.502 [initandlisten] connection accepted from 127.0.0.2:48626 #43 (3 connections now open) m31000| Wed Jan 16 18:21:46.502 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 686, "optime" : { "t" : 1358378507000, "i" : 3636 }, "optimeDate" : ISODate("2013-01-16T23:21:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 675, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 675, "optime" : { "t" : 1358378503000, "i" : 2466 }, "optimeDate" : ISODate("2013-01-16T23:21:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:45Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 685, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 672, "optime" : { "t" : 1358378507000, "i" : 1347 }, "optimeDate" : ISODate("2013-01-16T23:21:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 672, "optime" : { "t" : 1358378503000, "i" : 2466 }, "optimeDate" : ISODate("2013-01-16T23:21:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 672, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 684, "optime" : { "t" : 1358378507000, "i" : 2822 }, "optimeDate" : ISODate("2013-01-16T23:21:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:21:47.461 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:21:47.970 [conn43] end connection 127.0.0.2:43087 (3 connections now open) m31002| Wed Jan 16 18:21:48.471 [initandlisten] connection accepted from 127.0.0.2:48825 #45 (4 connections now open) m31000| Wed Jan 16 18:21:48.502 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:21:49.984 [conn33] end connection 127.0.0.4:46780 (4 connections now open) m31000| Wed Jan 16 18:21:49.984 [initandlisten] connection accepted from 127.0.0.4:44349 #34 (5 connections now open) m31000| Wed Jan 16 18:21:50.503 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 690, "optime" : { "t" : 1358378511000, "i" : 4408 }, "optimeDate" : ISODate("2013-01-16T23:21:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 679, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 679, "optime" : { "t" : 1358378510000, "i" : 4990 }, "optimeDate" : ISODate("2013-01-16T23:21:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 689, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 676, "optime" : { "t" : 1358378511000, "i" : 768 }, "optimeDate" : ISODate("2013-01-16T23:21:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 676, "optime" : { "t" : 1358378507000, "i" : 3636 }, "optimeDate" : ISODate("2013-01-16T23:21:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 676, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 688, "optime" : { "t" : 1358378511000, "i" : 4408 }, "optimeDate" : ISODate("2013-01-16T23:21:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:21:52.462 [conn41] end connection 127.0.0.4:45716 (2 connections now open) m31000| Wed Jan 16 18:21:52.503 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:21:52.962 [initandlisten] connection accepted from 127.0.0.4:36440 #44 (3 connections now open) m31002| Wed Jan 16 18:21:52.962 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:21:54.503 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 694, "optime" : { "t" : 1358378515000, "i" : 6418 }, "optimeDate" : ISODate("2013-01-16T23:21:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 683, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 683, "optime" : { "t" : 1358378511000, "i" : 4408 }, "optimeDate" : ISODate("2013-01-16T23:21:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:53Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 693, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 680, "optime" : { "t" : 1358378515000, "i" : 532 }, "optimeDate" : ISODate("2013-01-16T23:21:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:55Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 680, "optime" : { "t" : 1358378511000, "i" : 4408 }, "optimeDate" : ISODate("2013-01-16T23:21:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 680, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 692, "optime" : { "t" : 1358378515000, "i" : 5709 }, "optimeDate" : ISODate("2013-01-16T23:21:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:21:56.504 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:21:57.963 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:21:58.504 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:21:59.214 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:236964 118ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 698, "optime" : { "t" : 1358378519000, "i" : 7969 }, "optimeDate" : ISODate("2013-01-16T23:21:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 687, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 687, "optime" : { "t" : 1358378515000, "i" : 6418 }, "optimeDate" : ISODate("2013-01-16T23:21:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 697, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 684, "optime" : { "t" : 1358378519000, "i" : 952 }, "optimeDate" : ISODate("2013-01-16T23:21:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:21:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 684, "optime" : { "t" : 1358378515000, "i" : 6418 }, "optimeDate" : ISODate("2013-01-16T23:21:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 684, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:21:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 696, "optime" : { "t" : 1358378519000, "i" : 6994 }, "optimeDate" : ISODate("2013-01-16T23:21:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:22:00.505 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:22:02.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:22:02.505 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:22:02.963 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:22:03.128 [conn44] end connection 127.0.0.3:58374 (3 connections now open) m31002| Wed Jan 16 18:22:03.128 [initandlisten] connection accepted from 127.0.0.3:38558 #46 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 703, "optime" : { "t" : 1358378524000, "i" : 531 }, "optimeDate" : ISODate("2013-01-16T23:22:04Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 692, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 692, "optime" : { "t" : 1358378523000, "i" : 4379 }, "optimeDate" : ISODate("2013-01-16T23:22:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:03Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 702, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 689, "optime" : { "t" : 1358378523000, "i" : 1514 }, "optimeDate" : ISODate("2013-01-16T23:22:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 689, "optime" : { "t" : 1358378523000, "i" : 8513 }, "optimeDate" : ISODate("2013-01-16T23:22:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 689, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:03Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 701, "optime" : { "t" : 1358378524000, "i" : 531 }, "optimeDate" : ISODate("2013-01-16T23:22:04Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:22:04.505 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:22:06.506 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:22:07.452 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:344789 172ms m31002| Wed Jan 16 18:22:07.964 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 707, "optime" : { "t" : 1358378528000, "i" : 1281 }, "optimeDate" : ISODate("2013-01-16T23:22:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 696, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 696, "optime" : { "t" : 1358378524000, "i" : 531 }, "optimeDate" : ISODate("2013-01-16T23:22:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:07Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 706, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 693, "optime" : { "t" : 1358378527000, "i" : 514 }, "optimeDate" : ISODate("2013-01-16T23:22:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:07Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 693, "optime" : { "t" : 1358378527000, "i" : 8602 }, "optimeDate" : ISODate("2013-01-16T23:22:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 693, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 705, "optime" : { "t" : 1358378528000, "i" : 1281 }, "optimeDate" : ISODate("2013-01-16T23:22:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:22:08.506 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:22:10.506 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:22:12.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 711, "optime" : { "t" : 1358378532000, "i" : 2669 }, "optimeDate" : ISODate("2013-01-16T23:22:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 700, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 700, "optime" : { "t" : 1358378528000, "i" : 1281 }, "optimeDate" : ISODate("2013-01-16T23:22:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:11Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 710, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 697, "optime" : { "t" : 1358378528000, "i" : 1281 }, "optimeDate" : ISODate("2013-01-16T23:22:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 697, "optime" : { "t" : 1358378531000, "i" : 7227 }, "optimeDate" : ISODate("2013-01-16T23:22:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 697, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 709, "optime" : { "t" : 1358378532000, "i" : 1756 }, "optimeDate" : ISODate("2013-01-16T23:22:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:22:12.507 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:22:12.965 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:22:14.507 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:22:16.507 [conn43] end connection 127.0.0.2:48626 (2 connections now open) m31001| Wed Jan 16 18:22:16.508 [initandlisten] connection accepted from 127.0.0.2:35783 #45 (3 connections now open) m31000| Wed Jan 16 18:22:16.508 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 715, "optime" : { "t" : 1358378536000, "i" : 4156 }, "optimeDate" : ISODate("2013-01-16T23:22:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 704, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 704, "optime" : { "t" : 1358378535000, "i" : 5638 }, "optimeDate" : ISODate("2013-01-16T23:22:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:15Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 714, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 701, "optime" : { "t" : 1358378532000, "i" : 2669 }, "optimeDate" : ISODate("2013-01-16T23:22:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:15Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 701, "optime" : { "t" : 1358378535000, "i" : 5744 }, "optimeDate" : ISODate("2013-01-16T23:22:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 701, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 713, "optime" : { "t" : 1358378536000, "i" : 4156 }, "optimeDate" : ISODate("2013-01-16T23:22:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:22:17.965 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:22:18.508 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:22:19.989 [conn34] end connection 127.0.0.4:44349 (4 connections now open) m31000| Wed Jan 16 18:22:19.990 [initandlisten] connection accepted from 127.0.0.4:41332 #35 (6 connections now open) m31000| Wed Jan 16 18:22:20.508 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 719, "optime" : { "t" : 1358378540000, "i" : 6268 }, "optimeDate" : ISODate("2013-01-16T23:22:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 708, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 708, "optime" : { "t" : 1358378536000, "i" : 4156 }, "optimeDate" : ISODate("2013-01-16T23:22:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:19Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:21:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 718, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 705, "optime" : { "t" : 1358378536000, "i" : 4156 }, "optimeDate" : ISODate("2013-01-16T23:22:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:20Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 705, "optime" : { "t" : 1358378539000, "i" : 3635 }, "optimeDate" : ISODate("2013-01-16T23:22:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 705, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 717, "optime" : { "t" : 1358378540000, "i" : 6224 }, "optimeDate" : ISODate("2013-01-16T23:22:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:22:22.509 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:22:22.966 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:22:24.509 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 723, "optime" : { "t" : 1358378544000, "i" : 7049 }, "optimeDate" : ISODate("2013-01-16T23:22:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 712, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 712, "optime" : { "t" : 1358378540000, "i" : 6268 }, "optimeDate" : ISODate("2013-01-16T23:22:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:23Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 722, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 709, "optime" : { "t" : 1358378540000, "i" : 6268 }, "optimeDate" : ISODate("2013-01-16T23:22:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 709, "optime" : { "t" : 1358378543000, "i" : 2842 }, "optimeDate" : ISODate("2013-01-16T23:22:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 709, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 721, "optime" : { "t" : 1358378544000, "i" : 6452 }, "optimeDate" : ISODate("2013-01-16T23:22:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:22:25.976 [conn45] end connection 127.0.0.2:48825 (3 connections now open) m31002| Wed Jan 16 18:22:26.476 [initandlisten] connection accepted from 127.0.0.2:48268 #47 (4 connections now open) m31000| Wed Jan 16 18:22:26.509 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:22:27.966 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:22:28.510 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 727, "optime" : { "t" : 1358378548000, "i" : 8223 }, "optimeDate" : ISODate("2013-01-16T23:22:28Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 716, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 716, "optime" : { "t" : 1358378544000, "i" : 7049 }, "optimeDate" : ISODate("2013-01-16T23:22:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:27Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 726, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 713, "optime" : { "t" : 1358378544000, "i" : 7049 }, "optimeDate" : ISODate("2013-01-16T23:22:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:27Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:28Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 713, "optime" : { "t" : 1358378547000, "i" : 1776 }, "optimeDate" : ISODate("2013-01-16T23:22:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 713, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 725, "optime" : { "t" : 1358378548000, "i" : 8223 }, "optimeDate" : ISODate("2013-01-16T23:22:28Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:22:30.467 [conn44] end connection 127.0.0.4:36440 (2 connections now open) m31000| Wed Jan 16 18:22:30.510 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:22:30.967 [initandlisten] connection accepted from 127.0.0.4:59412 #46 (3 connections now open) m31000| Wed Jan 16 18:22:32.510 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 732, "optime" : { "t" : 1358378553000, "i" : 461 }, "optimeDate" : ISODate("2013-01-16T23:22:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 721, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 721, "optime" : { "t" : 1358378548000, "i" : 8223 }, "optimeDate" : ISODate("2013-01-16T23:22:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:31Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 731, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 718, "optime" : { "t" : 1358378548000, "i" : 8223 }, "optimeDate" : ISODate("2013-01-16T23:22:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 718, "optime" : { "t" : 1358378551000, "i" : 119 }, "optimeDate" : ISODate("2013-01-16T23:22:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 718, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:31Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 730, "optime" : { "t" : 1358378553000, "i" : 461 }, "optimeDate" : ISODate("2013-01-16T23:22:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:22:33.134 [conn46] end connection 127.0.0.3:38558 (3 connections now open) m31002| Wed Jan 16 18:22:33.134 [initandlisten] connection accepted from 127.0.0.3:55716 #48 (4 connections now open) m31002| Wed Jan 16 18:22:33.467 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:22:34.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:22:34.511 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:22:36.511 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 736, "optime" : { "t" : 1358378557000, "i" : 1456 }, "optimeDate" : ISODate("2013-01-16T23:22:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 725, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 725, "optime" : { "t" : 1358378556000, "i" : 3321 }, "optimeDate" : ISODate("2013-01-16T23:22:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 735, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 722, "optime" : { "t" : 1358378557000, "i" : 727 }, "optimeDate" : ISODate("2013-01-16T23:22:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 722, "optime" : { "t" : 1358378553000, "i" : 461 }, "optimeDate" : ISODate("2013-01-16T23:22:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 722, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 734, "optime" : { "t" : 1358378557000, "i" : 1415 }, "optimeDate" : ISODate("2013-01-16T23:22:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:22:38.468 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:22:38.511 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:22:40.512 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 740, "optime" : { "t" : 1358378561000, "i" : 2724 }, "optimeDate" : ISODate("2013-01-16T23:22:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 729, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 729, "optime" : { "t" : 1358378557000, "i" : 1456 }, "optimeDate" : ISODate("2013-01-16T23:22:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:39Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 739, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 726, "optime" : { "t" : 1358378561000, "i" : 1135 }, "optimeDate" : ISODate("2013-01-16T23:22:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:41Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 726, "optime" : { "t" : 1358378557000, "i" : 1456 }, "optimeDate" : ISODate("2013-01-16T23:22:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 726, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:41Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 738, "optime" : { "t" : 1358378561000, "i" : 2679 }, "optimeDate" : ISODate("2013-01-16T23:22:41Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:22:42.512 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:22:43.469 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:22:44.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:22:44.512 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 744, "optime" : { "t" : 1358378565000, "i" : 4408 }, "optimeDate" : ISODate("2013-01-16T23:22:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 733, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 733, "optime" : { "t" : 1358378561000, "i" : 2724 }, "optimeDate" : ISODate("2013-01-16T23:22:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:43Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 743, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 730, "optime" : { "t" : 1358378565000, "i" : 1234 }, "optimeDate" : ISODate("2013-01-16T23:22:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 730, "optime" : { "t" : 1358378561000, "i" : 2724 }, "optimeDate" : ISODate("2013-01-16T23:22:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 730, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 742, "optime" : { "t" : 1358378565000, "i" : 4107 }, "optimeDate" : ISODate("2013-01-16T23:22:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:22:46.513 [conn45] end connection 127.0.0.2:35783 (2 connections now open) m31001| Wed Jan 16 18:22:46.513 [initandlisten] connection accepted from 127.0.0.2:52553 #47 (3 connections now open) m31000| Wed Jan 16 18:22:46.513 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:22:48.469 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:22:48.513 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 748, "optime" : { "t" : 1358378569000, "i" : 5911 }, "optimeDate" : ISODate("2013-01-16T23:22:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 737, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 737, "optime" : { "t" : 1358378568000, "i" : 3922 }, "optimeDate" : ISODate("2013-01-16T23:22:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 747, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 734, "optime" : { "t" : 1358378569000, "i" : 1202 }, "optimeDate" : ISODate("2013-01-16T23:22:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 734, "optime" : { "t" : 1358378565000, "i" : 4408 }, "optimeDate" : ISODate("2013-01-16T23:22:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 734, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 746, "optime" : { "t" : 1358378569000, "i" : 5911 }, "optimeDate" : ISODate("2013-01-16T23:22:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:22:49.995 [conn35] end connection 127.0.0.4:41332 (4 connections now open) m31000| Wed Jan 16 18:22:49.995 [initandlisten] connection accepted from 127.0.0.4:60191 #36 (5 connections now open) m31000| Wed Jan 16 18:22:50.514 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:22:52.514 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:22:53.470 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 752, "optime" : { "t" : 1358378573000, "i" : 7207 }, "optimeDate" : ISODate("2013-01-16T23:22:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 741, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 741, "optime" : { "t" : 1358378569000, "i" : 5911 }, "optimeDate" : ISODate("2013-01-16T23:22:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:51Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 751, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 738, "optime" : { "t" : 1358378573000, "i" : 1771 }, "optimeDate" : ISODate("2013-01-16T23:22:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:53Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 738, "optime" : { "t" : 1358378569000, "i" : 5911 }, "optimeDate" : ISODate("2013-01-16T23:22:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 738, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 750, "optime" : { "t" : 1358378573000, "i" : 7207 }, "optimeDate" : ISODate("2013-01-16T23:22:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:22:54.514 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:22:56.515 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 756, "optime" : { "t" : 1358378577000, "i" : 7666 }, "optimeDate" : ISODate("2013-01-16T23:22:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 745, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 745, "optime" : { "t" : 1358378573000, "i" : 7207 }, "optimeDate" : ISODate("2013-01-16T23:22:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 755, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 742, "optime" : { "t" : 1358378577000, "i" : 1159 }, "optimeDate" : ISODate("2013-01-16T23:22:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:22:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 742, "optime" : { "t" : 1358378573000, "i" : 7207 }, "optimeDate" : ISODate("2013-01-16T23:22:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 742, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:22:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 754, "optime" : { "t" : 1358378577000, "i" : 7189 }, "optimeDate" : ISODate("2013-01-16T23:22:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:22:58.471 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:22:58.515 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:23:00.516 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 761, "optime" : { "t" : 1358378582000, "i" : 81 }, "optimeDate" : ISODate("2013-01-16T23:23:02Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 750, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 750, "optime" : { "t" : 1358378581000, "i" : 4054 }, "optimeDate" : ISODate("2013-01-16T23:23:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:01Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 760, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 747, "optime" : { "t" : 1358378581000, "i" : 906 }, "optimeDate" : ISODate("2013-01-16T23:23:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 747, "optime" : { "t" : 1358378581000, "i" : 8515 }, "optimeDate" : ISODate("2013-01-16T23:23:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 747, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 759, "optime" : { "t" : 1358378582000, "i" : 81 }, "optimeDate" : ISODate("2013-01-16T23:23:02Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:23:02.516 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:23:03.139 [conn48] end connection 127.0.0.3:55716 (3 connections now open) m31002| Wed Jan 16 18:23:03.139 [initandlisten] connection accepted from 127.0.0.3:33063 #49 (4 connections now open) m31002| Wed Jan 16 18:23:03.471 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:23:03.981 [conn47] end connection 127.0.0.2:48268 (3 connections now open) m31002| Wed Jan 16 18:23:04.481 [initandlisten] connection accepted from 127.0.0.2:47645 #50 (4 connections now open) m31000| Wed Jan 16 18:23:04.516 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:23:06.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 765, "optime" : { "t" : 1358378586000, "i" : 926 }, "optimeDate" : ISODate("2013-01-16T23:23:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 754, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 754, "optime" : { "t" : 1358378582000, "i" : 81 }, "optimeDate" : ISODate("2013-01-16T23:23:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:05Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 764, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 751, "optime" : { "t" : 1358378585000, "i" : 488 }, "optimeDate" : ISODate("2013-01-16T23:23:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:05Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 751, "optime" : { "t" : 1358378585000, "i" : 8887 }, "optimeDate" : ISODate("2013-01-16T23:23:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 751, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 763, "optime" : { "t" : 1358378586000, "i" : 926 }, "optimeDate" : ISODate("2013-01-16T23:23:06Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:23:06.517 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:23:08.472 [conn46] end connection 127.0.0.4:59412 (2 connections now open) m31000| Wed Jan 16 18:23:08.517 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:23:08.972 [initandlisten] connection accepted from 127.0.0.4:33974 #48 (3 connections now open) m31002| Wed Jan 16 18:23:08.972 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 769, "optime" : { "t" : 1358378590000, "i" : 2450 }, "optimeDate" : ISODate("2013-01-16T23:23:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 758, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 758, "optime" : { "t" : 1358378589000, "i" : 2521 }, "optimeDate" : ISODate("2013-01-16T23:23:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:09Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 768, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 755, "optime" : { "t" : 1358378586000, "i" : 926 }, "optimeDate" : ISODate("2013-01-16T23:23:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 755, "optime" : { "t" : 1358378589000, "i" : 7516 }, "optimeDate" : ISODate("2013-01-16T23:23:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 755, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:09Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 767, "optime" : { "t" : 1358378590000, "i" : 2450 }, "optimeDate" : ISODate("2013-01-16T23:23:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:23:10.518 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:23:12.518 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:23:13.973 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 773, "optime" : { "t" : 1358378594000, "i" : 4062 }, "optimeDate" : ISODate("2013-01-16T23:23:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 762, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 762, "optime" : { "t" : 1358378590000, "i" : 2450 }, "optimeDate" : ISODate("2013-01-16T23:23:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:13Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 772, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 759, "optime" : { "t" : 1358378590000, "i" : 2450 }, "optimeDate" : ISODate("2013-01-16T23:23:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:13Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 759, "optime" : { "t" : 1358378593000, "i" : 5805 }, "optimeDate" : ISODate("2013-01-16T23:23:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 759, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 771, "optime" : { "t" : 1358378594000, "i" : 4062 }, "optimeDate" : ISODate("2013-01-16T23:23:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:23:14.518 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:23:16.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 18:23:16.519 [conn47] end connection 127.0.0.2:52553 (2 connections now open) m31001| Wed Jan 16 18:23:16.519 [initandlisten] connection accepted from 127.0.0.2:33553 #49 (3 connections now open) m31000| Wed Jan 16 18:23:16.519 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:23:18.519 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 777, "optime" : { "t" : 1358378598000, "i" : 5231 }, "optimeDate" : ISODate("2013-01-16T23:23:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 766, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 766, "optime" : { "t" : 1358378594000, "i" : 4062 }, "optimeDate" : ISODate("2013-01-16T23:23:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:17Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 776, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 763, "optime" : { "t" : 1358378594000, "i" : 4062 }, "optimeDate" : ISODate("2013-01-16T23:23:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 763, "optime" : { "t" : 1358378597000, "i" : 4768 }, "optimeDate" : ISODate("2013-01-16T23:23:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 763, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 775, "optime" : { "t" : 1358378598000, "i" : 5231 }, "optimeDate" : ISODate("2013-01-16T23:23:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:23:18.974 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:23:19.999 [conn36] end connection 127.0.0.4:60191 (4 connections now open) m31000| Wed Jan 16 18:23:19.999 [initandlisten] connection accepted from 127.0.0.4:41885 #37 (5 connections now open) m31000| Wed Jan 16 18:23:20.520 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:23:22.071 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:327433 163ms m31000| Wed Jan 16 18:23:22.363 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:443379 221ms m31000| Wed Jan 16 18:23:22.520 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 781, "optime" : { "t" : 1358378602000, "i" : 7516 }, "optimeDate" : ISODate("2013-01-16T23:23:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 770, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 770, "optime" : { "t" : 1358378601000, "i" : 2480 }, "optimeDate" : ISODate("2013-01-16T23:23:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:21Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:22:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 780, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 767, "optime" : { "t" : 1358378598000, "i" : 5231 }, "optimeDate" : ISODate("2013-01-16T23:23:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:21Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 767, "optime" : { "t" : 1358378601000, "i" : 2483 }, "optimeDate" : ISODate("2013-01-16T23:23:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 767, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 779, "optime" : { "t" : 1358378602000, "i" : 7354 }, "optimeDate" : ISODate("2013-01-16T23:23:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:23:23.974 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:23:24.520 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:23:26.521 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 785, "optime" : { "t" : 1358378606000, "i" : 6661 }, "optimeDate" : ISODate("2013-01-16T23:23:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 774, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 774, "optime" : { "t" : 1358378602000, "i" : 7516 }, "optimeDate" : ISODate("2013-01-16T23:23:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:25Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 784, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 771, "optime" : { "t" : 1358378602000, "i" : 7516 }, "optimeDate" : ISODate("2013-01-16T23:23:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:26Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 771, "optime" : { "t" : 1358378605000, "i" : 3321 }, "optimeDate" : ISODate("2013-01-16T23:23:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 771, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 783, "optime" : { "t" : 1358378606000, "i" : 6625 }, "optimeDate" : ISODate("2013-01-16T23:23:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:23:28.521 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:23:28.975 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:23:30.521 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 789, "optime" : { "t" : 1358378610000, "i" : 8436 }, "optimeDate" : ISODate("2013-01-16T23:23:30Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 778, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 778, "optime" : { "t" : 1358378606000, "i" : 6661 }, "optimeDate" : ISODate("2013-01-16T23:23:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:29Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 788, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 775, "optime" : { "t" : 1358378606000, "i" : 6661 }, "optimeDate" : ISODate("2013-01-16T23:23:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 775, "optime" : { "t" : 1358378609000, "i" : 1563 }, "optimeDate" : ISODate("2013-01-16T23:23:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 775, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:29Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 787, "optime" : { "t" : 1358378610000, "i" : 8318 }, "optimeDate" : ISODate("2013-01-16T23:23:30Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:23:32.522 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:23:33.145 [conn49] end connection 127.0.0.3:33063 (3 connections now open) m31002| Wed Jan 16 18:23:33.145 [initandlisten] connection accepted from 127.0.0.3:35439 #51 (4 connections now open) m31002| Wed Jan 16 18:23:33.976 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:23:34.522 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 794, "optime" : { "t" : 1358378615000, "i" : 1462 }, "optimeDate" : ISODate("2013-01-16T23:23:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 783, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 783, "optime" : { "t" : 1358378614000, "i" : 4111 }, "optimeDate" : ISODate("2013-01-16T23:23:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 793, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 780, "optime" : { "t" : 1358378615000, "i" : 1275 }, "optimeDate" : ISODate("2013-01-16T23:23:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 780, "optime" : { "t" : 1358378613000, "i" : 286 }, "optimeDate" : ISODate("2013-01-16T23:23:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 780, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:35Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 792, "optime" : { "t" : 1358378615000, "i" : 1462 }, "optimeDate" : ISODate("2013-01-16T23:23:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:23:36.522 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:23:38.003 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:23:38.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:23:38.523 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 798, "optime" : { "t" : 1358378619000, "i" : 3712 }, "optimeDate" : ISODate("2013-01-16T23:23:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 787, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 787, "optime" : { "t" : 1358378615000, "i" : 1462 }, "optimeDate" : ISODate("2013-01-16T23:23:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:38Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 797, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 784, "optime" : { "t" : 1358378619000, "i" : 1197 }, "optimeDate" : ISODate("2013-01-16T23:23:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 784, "optime" : { "t" : 1358378615000, "i" : 1462 }, "optimeDate" : ISODate("2013-01-16T23:23:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 784, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:39Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 796, "optime" : { "t" : 1358378619000, "i" : 3712 }, "optimeDate" : ISODate("2013-01-16T23:23:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:23:40.523 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:23:41.986 [conn50] end connection 127.0.0.2:47645 (3 connections now open) m31002| Wed Jan 16 18:23:42.486 [initandlisten] connection accepted from 127.0.0.2:42849 #52 (4 connections now open) m31000| Wed Jan 16 18:23:42.523 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 802, "optime" : { "t" : 1358378623000, "i" : 5160 }, "optimeDate" : ISODate("2013-01-16T23:23:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 791, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 791, "optime" : { "t" : 1358378622000, "i" : 193 }, "optimeDate" : ISODate("2013-01-16T23:23:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 801, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 788, "optime" : { "t" : 1358378623000, "i" : 1418 }, "optimeDate" : ISODate("2013-01-16T23:23:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 788, "optime" : { "t" : 1358378619000, "i" : 3712 }, "optimeDate" : ISODate("2013-01-16T23:23:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 788, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:43Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 800, "optime" : { "t" : 1358378623000, "i" : 4822 }, "optimeDate" : ISODate("2013-01-16T23:23:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:23:43.977 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:23:44.524 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:23:46.477 [conn48] end connection 127.0.0.4:33974 (2 connections now open) m31001| Wed Jan 16 18:23:46.524 [conn49] end connection 127.0.0.2:33553 (1 connection now open) m31001| Wed Jan 16 18:23:46.524 [initandlisten] connection accepted from 127.0.0.2:48213 #50 (2 connections now open) m31000| Wed Jan 16 18:23:46.525 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:23:46.977 [initandlisten] connection accepted from 127.0.0.4:52997 #51 (3 connections now open) m31000| Wed Jan 16 18:23:47.358 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:260203 130ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 806, "optime" : { "t" : 1358378627000, "i" : 6047 }, "optimeDate" : ISODate("2013-01-16T23:23:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 795, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 795, "optime" : { "t" : 1358378627000, "i" : 3426 }, "optimeDate" : ISODate("2013-01-16T23:23:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 805, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 792, "optime" : { "t" : 1358378627000, "i" : 1134 }, "optimeDate" : ISODate("2013-01-16T23:23:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 792, "optime" : { "t" : 1358378623000, "i" : 5160 }, "optimeDate" : ISODate("2013-01-16T23:23:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:47Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 792, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 804, "optime" : { "t" : 1358378627000, "i" : 6047 }, "optimeDate" : ISODate("2013-01-16T23:23:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:23:48.005 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:23:48.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:23:48.525 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:23:50.005 [conn37] end connection 127.0.0.4:41885 (4 connections now open) m31000| Wed Jan 16 18:23:50.005 [initandlisten] connection accepted from 127.0.0.4:57199 #38 (5 connections now open) m31000| Wed Jan 16 18:23:50.525 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 810, "optime" : { "t" : 1358378631000, "i" : 7336 }, "optimeDate" : ISODate("2013-01-16T23:23:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 799, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 799, "optime" : { "t" : 1358378627000, "i" : 6047 }, "optimeDate" : ISODate("2013-01-16T23:23:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:50Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 809, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 796, "optime" : { "t" : 1358378631000, "i" : 1235 }, "optimeDate" : ISODate("2013-01-16T23:23:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 796, "optime" : { "t" : 1358378627000, "i" : 6047 }, "optimeDate" : ISODate("2013-01-16T23:23:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 796, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 808, "optime" : { "t" : 1358378631000, "i" : 7336 }, "optimeDate" : ISODate("2013-01-16T23:23:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:23:52.526 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:23:54.006 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:23:54.526 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 814, "optime" : { "t" : 1358378635000, "i" : 8896 }, "optimeDate" : ISODate("2013-01-16T23:23:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 803, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 803, "optime" : { "t" : 1358378634000, "i" : 483 }, "optimeDate" : ISODate("2013-01-16T23:23:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 813, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 800, "optime" : { "t" : 1358378635000, "i" : 802 }, "optimeDate" : ISODate("2013-01-16T23:23:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:23:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 800, "optime" : { "t" : 1358378631000, "i" : 7336 }, "optimeDate" : ISODate("2013-01-16T23:23:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 800, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 812, "optime" : { "t" : 1358378635000, "i" : 8896 }, "optimeDate" : ISODate("2013-01-16T23:23:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:23:56.527 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:23:58.527 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:23:59.479 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 819, "optime" : { "t" : 1358378640000, "i" : 1174 }, "optimeDate" : ISODate("2013-01-16T23:24:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 808, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 808, "optime" : { "t" : 1358378639000, "i" : 8681 }, "optimeDate" : ISODate("2013-01-16T23:23:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 818, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 805, "optime" : { "t" : 1358378639000, "i" : 1290 }, "optimeDate" : ISODate("2013-01-16T23:23:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:23:59Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 805, "optime" : { "t" : 1358378640000, "i" : 29 }, "optimeDate" : ISODate("2013-01-16T23:24:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 805, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 817, "optime" : { "t" : 1358378640000, "i" : 1174 }, "optimeDate" : ISODate("2013-01-16T23:24:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:24:00.527 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:24:02.528 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:24:03.150 [conn51] end connection 127.0.0.3:35439 (3 connections now open) m31002| Wed Jan 16 18:24:03.150 [initandlisten] connection accepted from 127.0.0.3:40396 #53 (4 connections now open) m31002| Wed Jan 16 18:24:04.008 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 823, "optime" : { "t" : 1358378644000, "i" : 2284 }, "optimeDate" : ISODate("2013-01-16T23:24:04Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 812, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 812, "optime" : { "t" : 1358378640000, "i" : 1174 }, "optimeDate" : ISODate("2013-01-16T23:24:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:04Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 822, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 809, "optime" : { "t" : 1358378643000, "i" : 7 }, "optimeDate" : ISODate("2013-01-16T23:24:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 809, "optime" : { "t" : 1358378643000, "i" : 7715 }, "optimeDate" : ISODate("2013-01-16T23:24:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 809, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:03Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 821, "optime" : { "t" : 1358378644000, "i" : 2284 }, "optimeDate" : ISODate("2013-01-16T23:24:04Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:24:04.528 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:24:06.529 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:24:07.409 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:256572 128ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 827, "optime" : { "t" : 1358378648000, "i" : 3286 }, "optimeDate" : ISODate("2013-01-16T23:24:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 816, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 816, "optime" : { "t" : 1358378647000, "i" : 2214 }, "optimeDate" : ISODate("2013-01-16T23:24:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:08Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 826, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 813, "optime" : { "t" : 1358378644000, "i" : 2284 }, "optimeDate" : ISODate("2013-01-16T23:24:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 813, "optime" : { "t" : 1358378648000, "i" : 47 }, "optimeDate" : ISODate("2013-01-16T23:24:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 813, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:07Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 825, "optime" : { "t" : 1358378648000, "i" : 3059 }, "optimeDate" : ISODate("2013-01-16T23:24:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:24:08.529 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:24:09.480 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:24:10.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:24:10.529 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 831, "optime" : { "t" : 1358378652000, "i" : 4507 }, "optimeDate" : ISODate("2013-01-16T23:24:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 820, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 820, "optime" : { "t" : 1358378652000, "i" : 3729 }, "optimeDate" : ISODate("2013-01-16T23:24:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 830, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 817, "optime" : { "t" : 1358378648000, "i" : 3286 }, "optimeDate" : ISODate("2013-01-16T23:24:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:11Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 817, "optime" : { "t" : 1358378652000, "i" : 64 }, "optimeDate" : ISODate("2013-01-16T23:24:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 817, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 829, "optime" : { "t" : 1358378652000, "i" : 4507 }, "optimeDate" : ISODate("2013-01-16T23:24:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:24:12.530 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:24:14.009 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:24:14.530 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:24:16.530 [conn50] end connection 127.0.0.2:48213 (2 connections now open) m31001| Wed Jan 16 18:24:16.530 [initandlisten] connection accepted from 127.0.0.2:44326 #52 (3 connections now open) m31000| Wed Jan 16 18:24:16.531 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 835, "optime" : { "t" : 1358378656000, "i" : 6123 }, "optimeDate" : ISODate("2013-01-16T23:24:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 824, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 824, "optime" : { "t" : 1358378652000, "i" : 4507 }, "optimeDate" : ISODate("2013-01-16T23:24:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 834, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 821, "optime" : { "t" : 1358378652000, "i" : 4507 }, "optimeDate" : ISODate("2013-01-16T23:24:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 821, "optime" : { "t" : 1358378656000, "i" : 147 }, "optimeDate" : ISODate("2013-01-16T23:24:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 821, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:15Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 833, "optime" : { "t" : 1358378656000, "i" : 6123 }, "optimeDate" : ISODate("2013-01-16T23:24:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:24:18.531 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:24:19.482 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:24:19.991 [conn52] end connection 127.0.0.2:42849 (3 connections now open) m31001| Wed Jan 16 18:24:20.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:24:20.010 [conn38] end connection 127.0.0.4:57199 (4 connections now open) m31000| Wed Jan 16 18:24:20.010 [initandlisten] connection accepted from 127.0.0.4:45868 #39 (5 connections now open) m31002| Wed Jan 16 18:24:20.491 [initandlisten] connection accepted from 127.0.0.2:50628 #54 (4 connections now open) m31000| Wed Jan 16 18:24:20.531 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 839, "optime" : { "t" : 1358378660000, "i" : 7067 }, "optimeDate" : ISODate("2013-01-16T23:24:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 828, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 828, "optime" : { "t" : 1358378660000, "i" : 4029 }, "optimeDate" : ISODate("2013-01-16T23:24:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 838, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 825, "optime" : { "t" : 1358378656000, "i" : 6123 }, "optimeDate" : ISODate("2013-01-16T23:24:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:19Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 825, "optime" : { "t" : 1358378659000, "i" : 2932 }, "optimeDate" : ISODate("2013-01-16T23:24:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 825, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 837, "optime" : { "t" : 1358378660000, "i" : 6735 }, "optimeDate" : ISODate("2013-01-16T23:24:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:24:22.532 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:24:24.011 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:24:24.482 [conn51] end connection 127.0.0.4:52997 (2 connections now open) m31000| Wed Jan 16 18:24:24.532 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 843, "optime" : { "t" : 1358378664000, "i" : 8336 }, "optimeDate" : ISODate("2013-01-16T23:24:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 832, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 832, "optime" : { "t" : 1358378660000, "i" : 7067 }, "optimeDate" : ISODate("2013-01-16T23:24:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:24Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 842, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 829, "optime" : { "t" : 1358378660000, "i" : 7067 }, "optimeDate" : ISODate("2013-01-16T23:24:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 829, "optime" : { "t" : 1358378664000, "i" : 53 }, "optimeDate" : ISODate("2013-01-16T23:24:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 829, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 841, "optime" : { "t" : 1358378664000, "i" : 8336 }, "optimeDate" : ISODate("2013-01-16T23:24:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:24:24.983 [initandlisten] connection accepted from 127.0.0.4:49119 #53 (3 connections now open) m31000| Wed Jan 16 18:24:26.532 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:24:28.532 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 848, "optime" : { "t" : 1358378669000, "i" : 1124 }, "optimeDate" : ISODate("2013-01-16T23:24:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 837, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 837, "optime" : { "t" : 1358378667000, "i" : 38 }, "optimeDate" : ISODate("2013-01-16T23:24:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:28Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:23:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 847, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 834, "optime" : { "t" : 1358378664000, "i" : 8336 }, "optimeDate" : ISODate("2013-01-16T23:24:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:27Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 834, "optime" : { "t" : 1358378667000, "i" : 91 }, "optimeDate" : ISODate("2013-01-16T23:24:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 834, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 846, "optime" : { "t" : 1358378669000, "i" : 1124 }, "optimeDate" : ISODate("2013-01-16T23:24:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:24:29.984 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:24:30.533 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:24:32.533 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:24:33.156 [conn53] end connection 127.0.0.3:40396 (3 connections now open) m31002| Wed Jan 16 18:24:33.156 [initandlisten] connection accepted from 127.0.0.3:33426 #55 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 852, "optime" : { "t" : 1358378673000, "i" : 2494 }, "optimeDate" : ISODate("2013-01-16T23:24:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 841, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 841, "optime" : { "t" : 1358378672000, "i" : 7410 }, "optimeDate" : ISODate("2013-01-16T23:24:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 851, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 838, "optime" : { "t" : 1358378673000, "i" : 1411 }, "optimeDate" : ISODate("2013-01-16T23:24:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 838, "optime" : { "t" : 1358378669000, "i" : 1124 }, "optimeDate" : ISODate("2013-01-16T23:24:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 838, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:33Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 850, "optime" : { "t" : 1358378673000, "i" : 2494 }, "optimeDate" : ISODate("2013-01-16T23:24:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:24:34.012 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:24:34.533 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:24:36.534 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 856, "optime" : { "t" : 1358378677000, "i" : 4397 }, "optimeDate" : ISODate("2013-01-16T23:24:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 845, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 845, "optime" : { "t" : 1358378673000, "i" : 2494 }, "optimeDate" : ISODate("2013-01-16T23:24:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:36Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 855, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 842, "optime" : { "t" : 1358378677000, "i" : 1440 }, "optimeDate" : ISODate("2013-01-16T23:24:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 842, "optime" : { "t" : 1358378673000, "i" : 2494 }, "optimeDate" : ISODate("2013-01-16T23:24:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 842, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 854, "optime" : { "t" : 1358378677000, "i" : 4397 }, "optimeDate" : ISODate("2013-01-16T23:24:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:24:38.534 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:24:39.985 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:24:40.534 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 860, "optime" : { "t" : 1358378681000, "i" : 5693 }, "optimeDate" : ISODate("2013-01-16T23:24:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 849, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 849, "optime" : { "t" : 1358378680000, "i" : 34 }, "optimeDate" : ISODate("2013-01-16T23:24:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 859, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 846, "optime" : { "t" : 1358378681000, "i" : 1016 }, "optimeDate" : ISODate("2013-01-16T23:24:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:41Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 846, "optime" : { "t" : 1358378677000, "i" : 4397 }, "optimeDate" : ISODate("2013-01-16T23:24:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 846, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:41Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 858, "optime" : { "t" : 1358378681000, "i" : 5446 }, "optimeDate" : ISODate("2013-01-16T23:24:41Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:24:42.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:24:42.535 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:24:44.014 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:24:44.535 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 864, "optime" : { "t" : 1358378685000, "i" : 7359 }, "optimeDate" : ISODate("2013-01-16T23:24:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 853, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 853, "optime" : { "t" : 1358378685000, "i" : 4039 }, "optimeDate" : ISODate("2013-01-16T23:24:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 863, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 850, "optime" : { "t" : 1358378685000, "i" : 1227 }, "optimeDate" : ISODate("2013-01-16T23:24:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 850, "optime" : { "t" : 1358378681000, "i" : 5693 }, "optimeDate" : ISODate("2013-01-16T23:24:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:45Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 850, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 862, "optime" : { "t" : 1358378685000, "i" : 6907 }, "optimeDate" : ISODate("2013-01-16T23:24:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:24:46.535 [conn52] end connection 127.0.0.2:44326 (2 connections now open) m31001| Wed Jan 16 18:24:46.536 [initandlisten] connection accepted from 127.0.0.2:52241 #54 (3 connections now open) m31000| Wed Jan 16 18:24:46.536 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:24:48.536 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 868, "optime" : { "t" : 1358378689000, "i" : 8568 }, "optimeDate" : ISODate("2013-01-16T23:24:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 857, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 857, "optime" : { "t" : 1358378685000, "i" : 7359 }, "optimeDate" : ISODate("2013-01-16T23:24:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:48Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 867, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 854, "optime" : { "t" : 1358378689000, "i" : 594 }, "optimeDate" : ISODate("2013-01-16T23:24:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 854, "optime" : { "t" : 1358378685000, "i" : 7359 }, "optimeDate" : ISODate("2013-01-16T23:24:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 854, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 866, "optime" : { "t" : 1358378689000, "i" : 8568 }, "optimeDate" : ISODate("2013-01-16T23:24:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:24:49.986 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:24:50.015 [conn39] end connection 127.0.0.4:45868 (4 connections now open) m31000| Wed Jan 16 18:24:50.015 [initandlisten] connection accepted from 127.0.0.4:51173 #40 (5 connections now open) m31000| Wed Jan 16 18:24:50.537 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:24:52.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:24:52.537 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:24:54.016 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 873, "optime" : { "t" : 1358378694000, "i" : 1060 }, "optimeDate" : ISODate("2013-01-16T23:24:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 862, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 862, "optime" : { "t" : 1358378692000, "i" : 505 }, "optimeDate" : ISODate("2013-01-16T23:24:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:54Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 872, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 859, "optime" : { "t" : 1358378693000, "i" : 1333 }, "optimeDate" : ISODate("2013-01-16T23:24:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 859, "optime" : { "t" : 1358378693000, "i" : 8434 }, "optimeDate" : ISODate("2013-01-16T23:24:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 859, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:53Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 871, "optime" : { "t" : 1358378694000, "i" : 1060 }, "optimeDate" : ISODate("2013-01-16T23:24:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:24:54.537 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:24:56.538 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:24:57.326 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:300152 150ms m31002| Wed Jan 16 18:24:57.996 [conn54] end connection 127.0.0.2:50628 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 877, "optime" : { "t" : 1358378698000, "i" : 1579 }, "optimeDate" : ISODate("2013-01-16T23:24:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 866, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 866, "optime" : { "t" : 1358378694000, "i" : 1060 }, "optimeDate" : ISODate("2013-01-16T23:24:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:58Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 876, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 863, "optime" : { "t" : 1358378697000, "i" : 533 }, "optimeDate" : ISODate("2013-01-16T23:24:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:24:57Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:24:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 863, "optime" : { "t" : 1358378698000, "i" : 18 }, "optimeDate" : ISODate("2013-01-16T23:24:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 863, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 875, "optime" : { "t" : 1358378698000, "i" : 1579 }, "optimeDate" : ISODate("2013-01-16T23:24:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:24:58.496 [initandlisten] connection accepted from 127.0.0.2:48114 #56 (4 connections now open) m31000| Wed Jan 16 18:24:58.538 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:24:59.988 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:25:00.538 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 881, "optime" : { "t" : 1358378702000, "i" : 2872 }, "optimeDate" : ISODate("2013-01-16T23:25:02Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 870, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 870, "optime" : { "t" : 1358378698000, "i" : 1579 }, "optimeDate" : ISODate("2013-01-16T23:24:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:02Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 880, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 867, "optime" : { "t" : 1358378698000, "i" : 1579 }, "optimeDate" : ISODate("2013-01-16T23:24:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 867, "optime" : { "t" : 1358378702000, "i" : 107 }, "optimeDate" : ISODate("2013-01-16T23:25:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 867, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:24:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 879, "optime" : { "t" : 1358378702000, "i" : 2872 }, "optimeDate" : ISODate("2013-01-16T23:25:02Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:25:02.488 [conn53] end connection 127.0.0.4:49119 (2 connections now open) m31000| Wed Jan 16 18:25:02.539 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:25:02.988 [initandlisten] connection accepted from 127.0.0.4:35396 #55 (3 connections now open) m31002| Wed Jan 16 18:25:03.161 [conn55] end connection 127.0.0.3:33426 (3 connections now open) m31002| Wed Jan 16 18:25:03.161 [initandlisten] connection accepted from 127.0.0.3:35992 #57 (4 connections now open) m31002| Wed Jan 16 18:25:04.018 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:25:04.539 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 885, "optime" : { "t" : 1358378706000, "i" : 4540 }, "optimeDate" : ISODate("2013-01-16T23:25:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 874, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 874, "optime" : { "t" : 1358378705000, "i" : 5407 }, "optimeDate" : ISODate("2013-01-16T23:25:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:06Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 884, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 871, "optime" : { "t" : 1358378702000, "i" : 2872 }, "optimeDate" : ISODate("2013-01-16T23:25:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:05Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 871, "optime" : { "t" : 1358378706000, "i" : 104 }, "optimeDate" : ISODate("2013-01-16T23:25:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 871, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 883, "optime" : { "t" : 1358378706000, "i" : 4046 }, "optimeDate" : ISODate("2013-01-16T23:25:06Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:25:06.539 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:25:08.540 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:25:10.018 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:25:10.540 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 889, "optime" : { "t" : 1358378710000, "i" : 5710 }, "optimeDate" : ISODate("2013-01-16T23:25:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 878, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 878, "optime" : { "t" : 1358378706000, "i" : 4540 }, "optimeDate" : ISODate("2013-01-16T23:25:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:10Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 888, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 875, "optime" : { "t" : 1358378706000, "i" : 4540 }, "optimeDate" : ISODate("2013-01-16T23:25:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:10Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 875, "optime" : { "t" : 1358378710000, "i" : 17 }, "optimeDate" : ISODate("2013-01-16T23:25:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 875, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 887, "optime" : { "t" : 1358378710000, "i" : 5294 }, "optimeDate" : ISODate("2013-01-16T23:25:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:25:12.540 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:25:14.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:25:14.541 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 893, "optime" : { "t" : 1358378714000, "i" : 6773 }, "optimeDate" : ISODate("2013-01-16T23:25:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 882, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 882, "optime" : { "t" : 1358378710000, "i" : 5710 }, "optimeDate" : ISODate("2013-01-16T23:25:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:14Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 892, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 879, "optime" : { "t" : 1358378710000, "i" : 5710 }, "optimeDate" : ISODate("2013-01-16T23:25:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 879, "optime" : { "t" : 1358378713000, "i" : 3226 }, "optimeDate" : ISODate("2013-01-16T23:25:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 879, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:13Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 891, "optime" : { "t" : 1358378714000, "i" : 5845 }, "optimeDate" : ISODate("2013-01-16T23:25:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:25:15.490 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:25:16.541 [conn54] end connection 127.0.0.2:52241 (2 connections now open) m31001| Wed Jan 16 18:25:16.541 [initandlisten] connection accepted from 127.0.0.2:48236 #56 (3 connections now open) m31000| Wed Jan 16 18:25:16.541 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:25:18.542 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 897, "optime" : { "t" : 1358378718000, "i" : 7944 }, "optimeDate" : ISODate("2013-01-16T23:25:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 886, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 886, "optime" : { "t" : 1358378718000, "i" : 4596 }, "optimeDate" : ISODate("2013-01-16T23:25:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 896, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 883, "optime" : { "t" : 1358378714000, "i" : 6773 }, "optimeDate" : ISODate("2013-01-16T23:25:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:17Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 883, "optime" : { "t" : 1358378718000, "i" : 522 }, "optimeDate" : ISODate("2013-01-16T23:25:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 883, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 895, "optime" : { "t" : 1358378718000, "i" : 7944 }, "optimeDate" : ISODate("2013-01-16T23:25:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:25:20.020 [conn40] end connection 127.0.0.4:51173 (4 connections now open) m31000| Wed Jan 16 18:25:20.020 [initandlisten] connection accepted from 127.0.0.4:41259 #41 (5 connections now open) m31002| Wed Jan 16 18:25:20.020 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:25:20.542 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:25:22.166 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:310360 155ms m31000| Wed Jan 16 18:25:22.542 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 902, "optime" : { "t" : 1358378723000, "i" : 30 }, "optimeDate" : ISODate("2013-01-16T23:25:23Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 891, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 891, "optime" : { "t" : 1358378718000, "i" : 7944 }, "optimeDate" : ISODate("2013-01-16T23:25:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:22Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 901, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 888, "optime" : { "t" : 1358378718000, "i" : 7944 }, "optimeDate" : ISODate("2013-01-16T23:25:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:22Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 888, "optime" : { "t" : 1358378722000, "i" : 91 }, "optimeDate" : ISODate("2013-01-16T23:25:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 888, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 900, "optime" : { "t" : 1358378723000, "i" : 30 }, "optimeDate" : ISODate("2013-01-16T23:25:23Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:25:24.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:25:24.543 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:25:25.491 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:25:26.543 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 906, "optime" : { "t" : 1358378727000, "i" : 1011 }, "optimeDate" : ISODate("2013-01-16T23:25:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 895, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 895, "optime" : { "t" : 1358378723000, "i" : 30 }, "optimeDate" : ISODate("2013-01-16T23:25:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:26Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 905, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 892, "optime" : { "t" : 1358378723000, "i" : 30 }, "optimeDate" : ISODate("2013-01-16T23:25:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:25Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 892, "optime" : { "t" : 1358378726000, "i" : 9 }, "optimeDate" : ISODate("2013-01-16T23:25:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 892, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 904, "optime" : { "t" : 1358378727000, "i" : 503 }, "optimeDate" : ISODate("2013-01-16T23:25:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:25:28.543 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:25:30.022 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:25:30.544 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 910, "optime" : { "t" : 1358378731000, "i" : 2705 }, "optimeDate" : ISODate("2013-01-16T23:25:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 899, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 899, "optime" : { "t" : 1358378730000, "i" : 7267 }, "optimeDate" : ISODate("2013-01-16T23:25:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 909, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 896, "optime" : { "t" : 1358378731000, "i" : 1336 }, "optimeDate" : ISODate("2013-01-16T23:25:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 896, "optime" : { "t" : 1358378727000, "i" : 1011 }, "optimeDate" : ISODate("2013-01-16T23:25:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 896, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:31Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 908, "optime" : { "t" : 1358378731000, "i" : 2705 }, "optimeDate" : ISODate("2013-01-16T23:25:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:25:32.544 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:25:33.167 [conn57] end connection 127.0.0.3:35992 (3 connections now open) m31002| Wed Jan 16 18:25:33.167 [initandlisten] connection accepted from 127.0.0.3:55811 #58 (4 connections now open) m31000| Wed Jan 16 18:25:34.544 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:25:35.493 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 914, "optime" : { "t" : 1358378735000, "i" : 4560 }, "optimeDate" : ISODate("2013-01-16T23:25:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 903, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 903, "optime" : { "t" : 1358378731000, "i" : 2705 }, "optimeDate" : ISODate("2013-01-16T23:25:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:34Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 913, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 900, "optime" : { "t" : 1358378735000, "i" : 1284 }, "optimeDate" : ISODate("2013-01-16T23:25:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:35Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 900, "optime" : { "t" : 1358378731000, "i" : 2705 }, "optimeDate" : ISODate("2013-01-16T23:25:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 900, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 912, "optime" : { "t" : 1358378735000, "i" : 4560 }, "optimeDate" : ISODate("2013-01-16T23:25:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:25:36.001 [conn56] end connection 127.0.0.2:48114 (3 connections now open) m31002| Wed Jan 16 18:25:36.501 [initandlisten] connection accepted from 127.0.0.2:44757 #59 (4 connections now open) m31000| Wed Jan 16 18:25:36.545 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:25:38.545 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 918, "optime" : { "t" : 1358378739000, "i" : 5716 }, "optimeDate" : ISODate("2013-01-16T23:25:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 907, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 907, "optime" : { "t" : 1358378738000, "i" : 3357 }, "optimeDate" : ISODate("2013-01-16T23:25:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 917, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 904, "optime" : { "t" : 1358378739000, "i" : 1325 }, "optimeDate" : ISODate("2013-01-16T23:25:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 904, "optime" : { "t" : 1358378735000, "i" : 4560 }, "optimeDate" : ISODate("2013-01-16T23:25:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 904, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:39Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 916, "optime" : { "t" : 1358378739000, "i" : 5219 }, "optimeDate" : ISODate("2013-01-16T23:25:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:25:40.024 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:25:40.493 [conn55] end connection 127.0.0.4:35396 (2 connections now open) m31000| Wed Jan 16 18:25:40.545 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:25:40.993 [initandlisten] connection accepted from 127.0.0.4:55396 #57 (3 connections now open) m31000| Wed Jan 16 18:25:42.546 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:25:43.262 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:210872 105ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 922, "optime" : { "t" : 1358378743000, "i" : 6778 }, "optimeDate" : ISODate("2013-01-16T23:25:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 911, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 911, "optime" : { "t" : 1358378739000, "i" : 5716 }, "optimeDate" : ISODate("2013-01-16T23:25:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:42Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 921, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 908, "optime" : { "t" : 1358378743000, "i" : 1316 }, "optimeDate" : ISODate("2013-01-16T23:25:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:43Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 908, "optime" : { "t" : 1358378739000, "i" : 5716 }, "optimeDate" : ISODate("2013-01-16T23:25:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 908, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 920, "optime" : { "t" : 1358378743000, "i" : 6778 }, "optimeDate" : ISODate("2013-01-16T23:25:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:25:44.546 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:25:45.994 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:25:46.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 18:25:46.547 [conn56] end connection 127.0.0.2:48236 (2 connections now open) m31001| Wed Jan 16 18:25:46.547 [initandlisten] connection accepted from 127.0.0.2:53951 #58 (3 connections now open) m31000| Wed Jan 16 18:25:46.547 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:25:47.462 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:294267 147ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 926, "optime" : { "t" : 1358378747000, "i" : 8240 }, "optimeDate" : ISODate("2013-01-16T23:25:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 915, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 915, "optime" : { "t" : 1358378743000, "i" : 6778 }, "optimeDate" : ISODate("2013-01-16T23:25:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 925, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 912, "optime" : { "t" : 1358378747000, "i" : 1093 }, "optimeDate" : ISODate("2013-01-16T23:25:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 912, "optime" : { "t" : 1358378743000, "i" : 6778 }, "optimeDate" : ISODate("2013-01-16T23:25:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 912, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 924, "optime" : { "t" : 1358378747000, "i" : 7726 }, "optimeDate" : ISODate("2013-01-16T23:25:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:25:48.547 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:25:50.025 [conn41] end connection 127.0.0.4:41259 (4 connections now open) m31000| Wed Jan 16 18:25:50.026 [initandlisten] connection accepted from 127.0.0.4:34145 #42 (5 connections now open) m31002| Wed Jan 16 18:25:50.026 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:25:50.548 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 931, "optime" : { "t" : 1358378752000, "i" : 1079 }, "optimeDate" : ISODate("2013-01-16T23:25:52Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 920, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 920, "optime" : { "t" : 1358378751000, "i" : 4254 }, "optimeDate" : ISODate("2013-01-16T23:25:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:52Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 930, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 917, "optime" : { "t" : 1358378751000, "i" : 1345 }, "optimeDate" : ISODate("2013-01-16T23:25:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 917, "optime" : { "t" : 1358378752000, "i" : 415 }, "optimeDate" : ISODate("2013-01-16T23:25:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 917, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 929, "optime" : { "t" : 1358378752000, "i" : 1079 }, "optimeDate" : ISODate("2013-01-16T23:25:52Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:25:52.548 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:25:54.548 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:25:55.996 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:25:56.006 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 935, "optime" : { "t" : 1358378756000, "i" : 1756 }, "optimeDate" : ISODate("2013-01-16T23:25:56Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 924, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 924, "optime" : { "t" : 1358378752000, "i" : 1079 }, "optimeDate" : ISODate("2013-01-16T23:25:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:56Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 934, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 921, "optime" : { "t" : 1358378755000, "i" : 61 }, "optimeDate" : ISODate("2013-01-16T23:25:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:55Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:25:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 921, "optime" : { "t" : 1358378756000, "i" : 599 }, "optimeDate" : ISODate("2013-01-16T23:25:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 921, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 933, "optime" : { "t" : 1358378756000, "i" : 1397 }, "optimeDate" : ISODate("2013-01-16T23:25:56Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:25:56.549 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:25:58.549 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:26:00.027 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 939, "optime" : { "t" : 1358378760000, "i" : 2910 }, "optimeDate" : ISODate("2013-01-16T23:26:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 928, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 928, "optime" : { "t" : 1358378756000, "i" : 1756 }, "optimeDate" : ISODate("2013-01-16T23:25:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 938, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 925, "optime" : { "t" : 1358378756000, "i" : 1756 }, "optimeDate" : ISODate("2013-01-16T23:25:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 925, "optime" : { "t" : 1358378759000, "i" : 7090 }, "optimeDate" : ISODate("2013-01-16T23:25:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 925, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:25:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 937, "optime" : { "t" : 1358378760000, "i" : 2810 }, "optimeDate" : ISODate("2013-01-16T23:26:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:26:00.550 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:26:02.550 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:26:03.172 [conn58] end connection 127.0.0.3:55811 (3 connections now open) m31002| Wed Jan 16 18:26:03.172 [initandlisten] connection accepted from 127.0.0.3:35413 #60 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 943, "optime" : { "t" : 1358378764000, "i" : 4260 }, "optimeDate" : ISODate("2013-01-16T23:26:04Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 932, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 932, "optime" : { "t" : 1358378764000, "i" : 8 }, "optimeDate" : ISODate("2013-01-16T23:26:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:04Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:25:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 942, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 929, "optime" : { "t" : 1358378760000, "i" : 2910 }, "optimeDate" : ISODate("2013-01-16T23:26:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:03Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 929, "optime" : { "t" : 1358378764000, "i" : 216 }, "optimeDate" : ISODate("2013-01-16T23:26:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 929, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 941, "optime" : { "t" : 1358378764000, "i" : 4260 }, "optimeDate" : ISODate("2013-01-16T23:26:04Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:26:04.551 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:26:05.997 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:26:06.551 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:26:08.551 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 947, "optime" : { "t" : 1358378768000, "i" : 5830 }, "optimeDate" : ISODate("2013-01-16T23:26:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 936, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 936, "optime" : { "t" : 1358378764000, "i" : 4260 }, "optimeDate" : ISODate("2013-01-16T23:26:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:08Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 946, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 933, "optime" : { "t" : 1358378764000, "i" : 4260 }, "optimeDate" : ISODate("2013-01-16T23:26:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:08Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 933, "optime" : { "t" : 1358378768000, "i" : 38 }, "optimeDate" : ISODate("2013-01-16T23:26:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 933, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 945, "optime" : { "t" : 1358378768000, "i" : 5735 }, "optimeDate" : ISODate("2013-01-16T23:26:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:26:10.029 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:26:10.552 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:26:12.552 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 951, "optime" : { "t" : 1358378772000, "i" : 7541 }, "optimeDate" : ISODate("2013-01-16T23:26:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 940, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 940, "optime" : { "t" : 1358378768000, "i" : 5830 }, "optimeDate" : ISODate("2013-01-16T23:26:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:12Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 950, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 937, "optime" : { "t" : 1358378768000, "i" : 5830 }, "optimeDate" : ISODate("2013-01-16T23:26:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 937, "optime" : { "t" : 1358378772000, "i" : 191 }, "optimeDate" : ISODate("2013-01-16T23:26:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 937, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 949, "optime" : { "t" : 1358378772000, "i" : 7541 }, "optimeDate" : ISODate("2013-01-16T23:26:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:26:14.006 [conn59] end connection 127.0.0.2:44757 (3 connections now open) m31002| Wed Jan 16 18:26:14.507 [initandlisten] connection accepted from 127.0.0.2:60114 #61 (4 connections now open) m31000| Wed Jan 16 18:26:14.552 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:26:15.998 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:26:16.552 [conn58] end connection 127.0.0.2:53951 (2 connections now open) m31001| Wed Jan 16 18:26:16.553 [initandlisten] connection accepted from 127.0.0.2:36252 #59 (3 connections now open) m31000| Wed Jan 16 18:26:16.553 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 955, "optime" : { "t" : 1358378776000, "i" : 8482 }, "optimeDate" : ISODate("2013-01-16T23:26:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 944, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 944, "optime" : { "t" : 1358378772000, "i" : 7541 }, "optimeDate" : ISODate("2013-01-16T23:26:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 954, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 941, "optime" : { "t" : 1358378772000, "i" : 7541 }, "optimeDate" : ISODate("2013-01-16T23:26:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:15Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 941, "optime" : { "t" : 1358378776000, "i" : 178 }, "optimeDate" : ISODate("2013-01-16T23:26:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 941, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 953, "optime" : { "t" : 1358378776000, "i" : 8273 }, "optimeDate" : ISODate("2013-01-16T23:26:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:26:18.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 18:26:18.498 [conn57] end connection 127.0.0.4:55396 (2 connections now open) m31000| Wed Jan 16 18:26:18.553 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:26:18.999 [initandlisten] connection accepted from 127.0.0.4:57544 #60 (3 connections now open) m31000| Wed Jan 16 18:26:20.030 [conn42] end connection 127.0.0.4:34145 (4 connections now open) m31000| Wed Jan 16 18:26:20.030 [initandlisten] connection accepted from 127.0.0.4:43139 #43 (5 connections now open) m31002| Wed Jan 16 18:26:20.030 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:26:20.553 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 960, "optime" : { "t" : 1358378781000, "i" : 1174 }, "optimeDate" : ISODate("2013-01-16T23:26:21Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 949, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 949, "optime" : { "t" : 1358378776000, "i" : 8482 }, "optimeDate" : ISODate("2013-01-16T23:26:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 959, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 946, "optime" : { "t" : 1358378776000, "i" : 8482 }, "optimeDate" : ISODate("2013-01-16T23:26:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 946, "optime" : { "t" : 1358378779000, "i" : 132 }, "optimeDate" : ISODate("2013-01-16T23:26:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 946, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:19Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 958, "optime" : { "t" : 1358378781000, "i" : 1174 }, "optimeDate" : ISODate("2013-01-16T23:26:21Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:26:22.554 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:26:24.554 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 964, "optime" : { "t" : 1358378785000, "i" : 2807 }, "optimeDate" : ISODate("2013-01-16T23:26:25Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 953, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 953, "optime" : { "t" : 1358378784000, "i" : 2873 }, "optimeDate" : ISODate("2013-01-16T23:26:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 963, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 950, "optime" : { "t" : 1358378785000, "i" : 1420 }, "optimeDate" : ISODate("2013-01-16T23:26:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:25Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 950, "optime" : { "t" : 1358378781000, "i" : 1174 }, "optimeDate" : ISODate("2013-01-16T23:26:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 950, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 962, "optime" : { "t" : 1358378785000, "i" : 2807 }, "optimeDate" : ISODate("2013-01-16T23:26:25Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:26:26.032 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:26:26.554 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:26:28.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:26:28.555 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 968, "optime" : { "t" : 1358378789000, "i" : 4361 }, "optimeDate" : ISODate("2013-01-16T23:26:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 957, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 957, "optime" : { "t" : 1358378785000, "i" : 2807 }, "optimeDate" : ISODate("2013-01-16T23:26:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:28Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 967, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 954, "optime" : { "t" : 1358378789000, "i" : 1575 }, "optimeDate" : ISODate("2013-01-16T23:26:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 954, "optime" : { "t" : 1358378785000, "i" : 2807 }, "optimeDate" : ISODate("2013-01-16T23:26:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 954, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:29Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 966, "optime" : { "t" : 1358378789000, "i" : 4361 }, "optimeDate" : ISODate("2013-01-16T23:26:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:26:30.555 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:26:31.501 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:26:32.555 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:26:33.177 [conn60] end connection 127.0.0.3:35413 (3 connections now open) m31002| Wed Jan 16 18:26:33.177 [initandlisten] connection accepted from 127.0.0.3:49253 #62 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 972, "optime" : { "t" : 1358378793000, "i" : 5571 }, "optimeDate" : ISODate("2013-01-16T23:26:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 961, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 961, "optime" : { "t" : 1358378789000, "i" : 4361 }, "optimeDate" : ISODate("2013-01-16T23:26:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:32Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 971, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 958, "optime" : { "t" : 1358378793000, "i" : 1305 }, "optimeDate" : ISODate("2013-01-16T23:26:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 958, "optime" : { "t" : 1358378789000, "i" : 4361 }, "optimeDate" : ISODate("2013-01-16T23:26:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 958, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:33Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 970, "optime" : { "t" : 1358378793000, "i" : 4747 }, "optimeDate" : ISODate("2013-01-16T23:26:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:26:34.555 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:26:36.033 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:26:36.556 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 976, "optime" : { "t" : 1358378797000, "i" : 6718 }, "optimeDate" : ISODate("2013-01-16T23:26:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 965, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 965, "optime" : { "t" : 1358378797000, "i" : 38 }, "optimeDate" : ISODate("2013-01-16T23:26:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 975, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 962, "optime" : { "t" : 1358378797000, "i" : 1621 }, "optimeDate" : ISODate("2013-01-16T23:26:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 962, "optime" : { "t" : 1358378793000, "i" : 5571 }, "optimeDate" : ISODate("2013-01-16T23:26:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:37Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 962, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 974, "optime" : { "t" : 1358378797000, "i" : 6718 }, "optimeDate" : ISODate("2013-01-16T23:26:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:26:38.556 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:26:40.557 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:26:41.502 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 980, "optime" : { "t" : 1358378801000, "i" : 7910 }, "optimeDate" : ISODate("2013-01-16T23:26:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 969, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 969, "optime" : { "t" : 1358378797000, "i" : 6718 }, "optimeDate" : ISODate("2013-01-16T23:26:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:40Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 979, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 966, "optime" : { "t" : 1358378801000, "i" : 1109 }, "optimeDate" : ISODate("2013-01-16T23:26:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:41Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 967, "optime" : { "t" : 1358378797000, "i" : 6718 }, "optimeDate" : ISODate("2013-01-16T23:26:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 967, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 979, "optime" : { "t" : 1358378801000, "i" : 7704 }, "optimeDate" : ISODate("2013-01-16T23:26:41Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:26:42.557 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:26:44.558 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:26:46.035 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 985, "optime" : { "t" : 1358378806000, "i" : 1555 }, "optimeDate" : ISODate("2013-01-16T23:26:46Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 974, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 974, "optime" : { "t" : 1358378801000, "i" : 7910 }, "optimeDate" : ISODate("2013-01-16T23:26:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:46Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 984, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 971, "optime" : { "t" : 1358378805000, "i" : 1516 }, "optimeDate" : ISODate("2013-01-16T23:26:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 971, "optime" : { "t" : 1358378805000, "i" : 8444 }, "optimeDate" : ISODate("2013-01-16T23:26:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 971, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 983, "optime" : { "t" : 1358378806000, "i" : 1555 }, "optimeDate" : ISODate("2013-01-16T23:26:46Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:26:46.558 [conn59] end connection 127.0.0.2:36252 (2 connections now open) m31001| Wed Jan 16 18:26:46.558 [initandlisten] connection accepted from 127.0.0.2:53518 #61 (3 connections now open) m31000| Wed Jan 16 18:26:46.558 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:26:48.559 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:26:50.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:26:50.036 [conn43] end connection 127.0.0.4:43139 (4 connections now open) m31000| Wed Jan 16 18:26:50.036 [initandlisten] connection accepted from 127.0.0.4:54256 #44 (6 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 989, "optime" : { "t" : 1358378810000, "i" : 3400 }, "optimeDate" : ISODate("2013-01-16T23:26:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 978, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 978, "optime" : { "t" : 1358378809000, "i" : 1935 }, "optimeDate" : ISODate("2013-01-16T23:26:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:50Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 988, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 975, "optime" : { "t" : 1358378806000, "i" : 1555 }, "optimeDate" : ISODate("2013-01-16T23:26:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 975, "optime" : { "t" : 1358378810000, "i" : 324 }, "optimeDate" : ISODate("2013-01-16T23:26:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 975, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 987, "optime" : { "t" : 1358378810000, "i" : 3066 }, "optimeDate" : ISODate("2013-01-16T23:26:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:26:50.559 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:26:51.503 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:26:52.012 [conn61] end connection 127.0.0.2:60114 (3 connections now open) m31002| Wed Jan 16 18:26:52.512 [initandlisten] connection accepted from 127.0.0.2:43170 #63 (4 connections now open) m31000| Wed Jan 16 18:26:52.559 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:26:54.560 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 993, "optime" : { "t" : 1358378814000, "i" : 4988 }, "optimeDate" : ISODate("2013-01-16T23:26:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 982, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 982, "optime" : { "t" : 1358378810000, "i" : 3400 }, "optimeDate" : ISODate("2013-01-16T23:26:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:54Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 992, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 979, "optime" : { "t" : 1358378810000, "i" : 3400 }, "optimeDate" : ISODate("2013-01-16T23:26:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:54Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 979, "optime" : { "t" : 1358378814000, "i" : 534 }, "optimeDate" : ISODate("2013-01-16T23:26:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 979, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 991, "optime" : { "t" : 1358378814000, "i" : 4988 }, "optimeDate" : ISODate("2013-01-16T23:26:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:26:56.037 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:26:56.504 [conn60] end connection 127.0.0.4:57544 (2 connections now open) m31000| Wed Jan 16 18:26:56.560 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:26:57.004 [initandlisten] connection accepted from 127.0.0.4:39095 #62 (3 connections now open) m31000| Wed Jan 16 18:26:58.560 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 997, "optime" : { "t" : 1358378818000, "i" : 6741 }, "optimeDate" : ISODate("2013-01-16T23:26:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 986, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 986, "optime" : { "t" : 1358378814000, "i" : 4988 }, "optimeDate" : ISODate("2013-01-16T23:26:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:58Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 996, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 983, "optime" : { "t" : 1358378814000, "i" : 4988 }, "optimeDate" : ISODate("2013-01-16T23:26:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:26:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 983, "optime" : { "t" : 1358378818000, "i" : 854 }, "optimeDate" : ISODate("2013-01-16T23:26:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 983, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:26:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 995, "optime" : { "t" : 1358378818000, "i" : 6741 }, "optimeDate" : ISODate("2013-01-16T23:26:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:27:00.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:27:00.561 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:27:02.005 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:27:02.433 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:384596 192ms m31000| Wed Jan 16 18:27:02.561 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1001, "optime" : { "t" : 1358378822000, "i" : 7881 }, "optimeDate" : ISODate("2013-01-16T23:27:02Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 990, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 990, "optime" : { "t" : 1358378822000, "i" : 4370 }, "optimeDate" : ISODate("2013-01-16T23:27:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1000, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 987, "optime" : { "t" : 1358378818000, "i" : 6741 }, "optimeDate" : ISODate("2013-01-16T23:26:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:02Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 987, "optime" : { "t" : 1358378822000, "i" : 484 }, "optimeDate" : ISODate("2013-01-16T23:27:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 987, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 999, "optime" : { "t" : 1358378822000, "i" : 7881 }, "optimeDate" : ISODate("2013-01-16T23:27:02Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:27:03.183 [conn62] end connection 127.0.0.3:49253 (3 connections now open) m31002| Wed Jan 16 18:27:03.183 [initandlisten] connection accepted from 127.0.0.3:42059 #64 (4 connections now open) m31000| Wed Jan 16 18:27:04.561 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:27:06.561 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:27:07.005 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1006, "optime" : { "t" : 1358378827000, "i" : 534 }, "optimeDate" : ISODate("2013-01-16T23:27:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 995, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 995, "optime" : { "t" : 1358378822000, "i" : 7881 }, "optimeDate" : ISODate("2013-01-16T23:27:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:06Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:26:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1005, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 992, "optime" : { "t" : 1358378822000, "i" : 7881 }, "optimeDate" : ISODate("2013-01-16T23:27:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:07Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 992, "optime" : { "t" : 1358378826000, "i" : 349 }, "optimeDate" : ISODate("2013-01-16T23:27:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 992, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1004, "optime" : { "t" : 1358378827000, "i" : 534 }, "optimeDate" : ISODate("2013-01-16T23:27:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:27:08.562 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:27:10.562 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1010, "optime" : { "t" : 1358378831000, "i" : 1682 }, "optimeDate" : ISODate("2013-01-16T23:27:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 999, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 999, "optime" : { "t" : 1358378827000, "i" : 534 }, "optimeDate" : ISODate("2013-01-16T23:27:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:10Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1009, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 996, "optime" : { "t" : 1358378827000, "i" : 534 }, "optimeDate" : ISODate("2013-01-16T23:27:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:09Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 996, "optime" : { "t" : 1358378830000, "i" : 40 }, "optimeDate" : ISODate("2013-01-16T23:27:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 996, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1008, "optime" : { "t" : 1358378831000, "i" : 1516 }, "optimeDate" : ISODate("2013-01-16T23:27:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:27:12.006 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:27:12.563 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:27:14.563 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1014, "optime" : { "t" : 1358378835000, "i" : 2911 }, "optimeDate" : ISODate("2013-01-16T23:27:15Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1003, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1003, "optime" : { "t" : 1358378835000, "i" : 236 }, "optimeDate" : ISODate("2013-01-16T23:27:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1013, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1000, "optime" : { "t" : 1358378835000, "i" : 1727 }, "optimeDate" : ISODate("2013-01-16T23:27:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1000, "optime" : { "t" : 1358378831000, "i" : 1682 }, "optimeDate" : ISODate("2013-01-16T23:27:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:15Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1000, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:15Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1012, "optime" : { "t" : 1358378835000, "i" : 2911 }, "optimeDate" : ISODate("2013-01-16T23:27:15Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:27:16.563 [conn61] end connection 127.0.0.2:53518 (2 connections now open) m31001| Wed Jan 16 18:27:16.563 [initandlisten] connection accepted from 127.0.0.2:42788 #63 (3 connections now open) m31000| Wed Jan 16 18:27:16.563 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:27:17.007 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:27:18.428 [FileAllocator] allocating new datafile /data/db/testReplSet-0/foo.4, filling with zeroes... m31000| Wed Jan 16 18:27:18.564 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:27:20.040 [conn44] end connection 127.0.0.4:54256 (4 connections now open) m31000| Wed Jan 16 18:27:20.040 [initandlisten] connection accepted from 127.0.0.4:44036 #45 (5 connections now open) m31000| Wed Jan 16 18:27:20.445 [FileAllocator] done allocating datafile /data/db/testReplSet-0/foo.4, size: 256MB, took 2.017 secs m31000| Wed Jan 16 18:27:20.445 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:2018208 2018ms m31002| Wed Jan 16 18:27:20.446 [FileAllocator] allocating new datafile /data/db/testReplSet-2/foo.4, filling with zeroes... m31000| Wed Jan 16 18:27:20.564 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1019, "optime" : { "t" : 1358378840000, "i" : 9187 }, "optimeDate" : ISODate("2013-01-16T23:27:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1008, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1008, "optime" : { "t" : 1358378838000, "i" : 813 }, "optimeDate" : ISODate("2013-01-16T23:27:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1018, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1005, "optime" : { "t" : 1358378838000, "i" : 813 }, "optimeDate" : ISODate("2013-01-16T23:27:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:19Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1005, "optime" : { "t" : 1358378838000, "i" : 813 }, "optimeDate" : ISODate("2013-01-16T23:27:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1005, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1017, "optime" : { "t" : 1358378838000, "i" : 813 }, "optimeDate" : ISODate("2013-01-16T23:27:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:27:22.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:27:22.007 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:27:22.564 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:27:24.565 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:27:24.631 [FileAllocator] done allocating datafile /data/db/testReplSet-2/foo.4, size: 256MB, took 4.184 secs m31000| Wed Jan 16 18:27:24.652 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:1445855 722ms STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:27:24.928 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:412517 206ms { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1024, "optime" : { "t" : 1358378845000, "i" : 5297 }, "optimeDate" : ISODate("2013-01-16T23:27:25Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1013, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1013, "optime" : { "t" : 1358378840000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:27:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1023, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1010, "optime" : { "t" : 1358378840000, "i" : 5002 }, "optimeDate" : ISODate("2013-01-16T23:27:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:25Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1010, "optime" : { "t" : 1358378843000, "i" : 602 }, "optimeDate" : ISODate("2013-01-16T23:27:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:25Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1010, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1022, "optime" : { "t" : 1358378844000, "i" : 213 }, "optimeDate" : ISODate("2013-01-16T23:27:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:27:26.565 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:27:27.008 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:27:28.565 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1028, "optime" : { "t" : 1358378849000, "i" : 3871 }, "optimeDate" : ISODate("2013-01-16T23:27:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1017, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1017, "optime" : { "t" : 1358378845000, "i" : 5297 }, "optimeDate" : ISODate("2013-01-16T23:27:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:28Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1027, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1014, "optime" : { "t" : 1358378849000, "i" : 1613 }, "optimeDate" : ISODate("2013-01-16T23:27:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1014, "optime" : { "t" : 1358378845000, "i" : 5297 }, "optimeDate" : ISODate("2013-01-16T23:27:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1014, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:29Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1026, "optime" : { "t" : 1358378849000, "i" : 3614 }, "optimeDate" : ISODate("2013-01-16T23:27:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:27:30.017 [conn63] end connection 127.0.0.2:43170 (3 connections now open) m31002| Wed Jan 16 18:27:30.517 [initandlisten] connection accepted from 127.0.0.2:51888 #65 (4 connections now open) m31000| Wed Jan 16 18:27:30.566 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:27:32.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:27:32.009 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:27:32.566 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:27:32.596 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:289090 144ms m31002| Wed Jan 16 18:27:33.188 [conn64] end connection 127.0.0.3:42059 (3 connections now open) m31002| Wed Jan 16 18:27:33.188 [initandlisten] connection accepted from 127.0.0.3:38025 #66 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1032, "optime" : { "t" : 1358378853000, "i" : 3852 }, "optimeDate" : ISODate("2013-01-16T23:27:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1021, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1021, "optime" : { "t" : 1358378853000, "i" : 37 }, "optimeDate" : ISODate("2013-01-16T23:27:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1031, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1018, "optime" : { "t" : 1358378853000, "i" : 1328 }, "optimeDate" : ISODate("2013-01-16T23:27:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1018, "optime" : { "t" : 1358378849000, "i" : 3871 }, "optimeDate" : ISODate("2013-01-16T23:27:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:33Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1018, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:33Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1030, "optime" : { "t" : 1358378853000, "i" : 2777 }, "optimeDate" : ISODate("2013-01-16T23:27:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:27:34.509 [conn62] end connection 127.0.0.4:39095 (2 connections now open) m31000| Wed Jan 16 18:27:34.566 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:27:35.009 [initandlisten] connection accepted from 127.0.0.4:56294 #64 (3 connections now open) m31000| Wed Jan 16 18:27:36.567 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:27:37.510 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1036, "optime" : { "t" : 1358378857000, "i" : 4926 }, "optimeDate" : ISODate("2013-01-16T23:27:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1025, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1025, "optime" : { "t" : 1358378853000, "i" : 3852 }, "optimeDate" : ISODate("2013-01-16T23:27:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:36Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1035, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1022, "optime" : { "t" : 1358378857000, "i" : 1235 }, "optimeDate" : ISODate("2013-01-16T23:27:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:37Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1022, "optime" : { "t" : 1358378853000, "i" : 3852 }, "optimeDate" : ISODate("2013-01-16T23:27:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1022, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1034, "optime" : { "t" : 1358378857000, "i" : 4926 }, "optimeDate" : ISODate("2013-01-16T23:27:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:27:38.567 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:27:40.567 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1040, "optime" : { "t" : 1358378861000, "i" : 7021 }, "optimeDate" : ISODate("2013-01-16T23:27:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1029, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1029, "optime" : { "t" : 1358378857000, "i" : 4926 }, "optimeDate" : ISODate("2013-01-16T23:27:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1039, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1026, "optime" : { "t" : 1358378861000, "i" : 1829 }, "optimeDate" : ISODate("2013-01-16T23:27:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:41Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1026, "optime" : { "t" : 1358378857000, "i" : 4926 }, "optimeDate" : ISODate("2013-01-16T23:27:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1026, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:41Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1038, "optime" : { "t" : 1358378861000, "i" : 7021 }, "optimeDate" : ISODate("2013-01-16T23:27:41Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:27:42.045 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:27:42.568 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:27:44.568 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1044, "optime" : { "t" : 1358378865000, "i" : 8481 }, "optimeDate" : ISODate("2013-01-16T23:27:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1033, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1033, "optime" : { "t" : 1358378865000, "i" : 4590 }, "optimeDate" : ISODate("2013-01-16T23:27:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1043, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1030, "optime" : { "t" : 1358378865000, "i" : 1474 }, "optimeDate" : ISODate("2013-01-16T23:27:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1030, "optime" : { "t" : 1358378861000, "i" : 7021 }, "optimeDate" : ISODate("2013-01-16T23:27:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:45Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1030, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1042, "optime" : { "t" : 1358378865000, "i" : 7820 }, "optimeDate" : ISODate("2013-01-16T23:27:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:27:46.569 [conn63] end connection 127.0.0.2:42788 (2 connections now open) m31001| Wed Jan 16 18:27:46.569 [initandlisten] connection accepted from 127.0.0.2:40317 #65 (3 connections now open) m31000| Wed Jan 16 18:27:46.569 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:27:47.511 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:27:48.569 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1049, "optime" : { "t" : 1358378870000, "i" : 36 }, "optimeDate" : ISODate("2013-01-16T23:27:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1038, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1038, "optime" : { "t" : 1358378865000, "i" : 8481 }, "optimeDate" : ISODate("2013-01-16T23:27:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:48Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1048, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1035, "optime" : { "t" : 1358378869000, "i" : 1789 }, "optimeDate" : ISODate("2013-01-16T23:27:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:50Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1035, "optime" : { "t" : 1358378865000, "i" : 8481 }, "optimeDate" : ISODate("2013-01-16T23:27:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1035, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1047, "optime" : { "t" : 1358378870000, "i" : 36 }, "optimeDate" : ISODate("2013-01-16T23:27:50Z"), "self" : true } ], "ok" : 1 } m31000| Wed Jan 16 18:27:50.046 [conn45] end connection 127.0.0.4:44036 (4 connections now open) m31000| Wed Jan 16 18:27:50.046 [initandlisten] connection accepted from 127.0.0.4:43813 #46 (5 connections now open) { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:27:50.570 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:27:52.047 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:27:52.570 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:27:54.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:27:54.571 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:27:54.749 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:3263276 1631ms m31000| Wed Jan 16 18:27:54.749 [conn10] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358377825000|1 } } cursorid:131228461816724 ntoreturn:0 keyUpdates:0 locks(micros) r:29 nreturned:1 reslen:117 1631ms STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:27:55.197 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:755804 377ms { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1054, "optime" : { "t" : 1358378875000, "i" : 7002 }, "optimeDate" : ISODate("2013-01-16T23:27:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1043, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1043, "optime" : { "t" : 1358378875000, "i" : 5441 }, "optimeDate" : ISODate("2013-01-16T23:27:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1053, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1040, "optime" : { "t" : 1358378874000, "i" : 2341 }, "optimeDate" : ISODate("2013-01-16T23:27:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1040, "optime" : { "t" : 1358378873000, "i" : 655 }, "optimeDate" : ISODate("2013-01-16T23:27:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:55Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1040, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1052, "optime" : { "t" : 1358378875000, "i" : 7002 }, "optimeDate" : ISODate("2013-01-16T23:27:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:27:56.571 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:27:57.513 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:27:58.572 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1058, "optime" : { "t" : 1358378879000, "i" : 6564 }, "optimeDate" : ISODate("2013-01-16T23:27:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1047, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1047, "optime" : { "t" : 1358378875000, "i" : 7002 }, "optimeDate" : ISODate("2013-01-16T23:27:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:58Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1057, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1044, "optime" : { "t" : 1358378879000, "i" : 1643 }, "optimeDate" : ISODate("2013-01-16T23:27:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:27:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1044, "optime" : { "t" : 1358378875000, "i" : 7002 }, "optimeDate" : ISODate("2013-01-16T23:27:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1044, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:27:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1056, "optime" : { "t" : 1358378879000, "i" : 6564 }, "optimeDate" : ISODate("2013-01-16T23:27:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:28:00.572 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:28:02.049 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:28:02.572 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:28:03.193 [conn66] end connection 127.0.0.3:38025 (3 connections now open) m31002| Wed Jan 16 18:28:03.193 [initandlisten] connection accepted from 127.0.0.3:44526 #67 (4 connections now open) m31000| Wed Jan 16 18:28:03.782 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:244411 122ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1062, "optime" : { "t" : 1358378883000, "i" : 7184 }, "optimeDate" : ISODate("2013-01-16T23:28:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1051, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1051, "optime" : { "t" : 1358378883000, "i" : 45 }, "optimeDate" : ISODate("2013-01-16T23:28:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1061, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1048, "optime" : { "t" : 1358378883000, "i" : 1387 }, "optimeDate" : ISODate("2013-01-16T23:28:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1048, "optime" : { "t" : 1358378879000, "i" : 6564 }, "optimeDate" : ISODate("2013-01-16T23:27:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:03Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1048, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:03Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1060, "optime" : { "t" : 1358378883000, "i" : 6730 }, "optimeDate" : ISODate("2013-01-16T23:28:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:28:04.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:28:04.573 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:28:06.573 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:28:07.514 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:28:07.954 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:212384 106ms { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1066, "optime" : { "t" : 1358378887000, "i" : 8180 }, "optimeDate" : ISODate("2013-01-16T23:28:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1055, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1055, "optime" : { "t" : 1358378883000, "i" : 7184 }, "optimeDate" : ISODate("2013-01-16T23:28:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:06Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1065, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1052, "optime" : { "t" : 1358378887000, "i" : 1328 }, "optimeDate" : ISODate("2013-01-16T23:28:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:07Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1052, "optime" : { "t" : 1358378883000, "i" : 7184 }, "optimeDate" : ISODate("2013-01-16T23:28:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1052, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1064, "optime" : { "t" : 1358378887000, "i" : 8180 }, "optimeDate" : ISODate("2013-01-16T23:28:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:28:08.022 [conn65] end connection 127.0.0.2:51888 (3 connections now open) m31002| Wed Jan 16 18:28:08.523 [initandlisten] connection accepted from 127.0.0.2:34988 #68 (4 connections now open) m31000| Wed Jan 16 18:28:08.574 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:28:10.574 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:28:12.050 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1071, "optime" : { "t" : 1358378892000, "i" : 1239 }, "optimeDate" : ISODate("2013-01-16T23:28:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1060, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1060, "optime" : { "t" : 1358378890000, "i" : 41 }, "optimeDate" : ISODate("2013-01-16T23:28:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:12Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:27:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1070, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1057, "optime" : { "t" : 1358378891000, "i" : 1354 }, "optimeDate" : ISODate("2013-01-16T23:28:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1057, "optime" : { "t" : 1358378892000, "i" : 1042 }, "optimeDate" : ISODate("2013-01-16T23:28:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1057, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1069, "optime" : { "t" : 1358378892000, "i" : 1239 }, "optimeDate" : ISODate("2013-01-16T23:28:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:28:12.514 [conn64] end connection 127.0.0.4:56294 (2 connections now open) m31000| Wed Jan 16 18:28:12.574 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:28:13.015 [initandlisten] connection accepted from 127.0.0.4:42900 #66 (3 connections now open) m31000| Wed Jan 16 18:28:14.575 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1075, "optime" : { "t" : 1358378896000, "i" : 1213 }, "optimeDate" : ISODate("2013-01-16T23:28:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1064, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1064, "optime" : { "t" : 1358378895000, "i" : 8773 }, "optimeDate" : ISODate("2013-01-16T23:28:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1074, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1061, "optime" : { "t" : 1358378895000, "i" : 660 }, "optimeDate" : ISODate("2013-01-16T23:28:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:15Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1061, "optime" : { "t" : 1358378896000, "i" : 355 }, "optimeDate" : ISODate("2013-01-16T23:28:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1061, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1073, "optime" : { "t" : 1358378896000, "i" : 1138 }, "optimeDate" : ISODate("2013-01-16T23:28:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:28:16.575 [conn65] end connection 127.0.0.2:40317 (2 connections now open) m31001| Wed Jan 16 18:28:16.575 [initandlisten] connection accepted from 127.0.0.2:44139 #67 (3 connections now open) m31000| Wed Jan 16 18:28:16.576 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:28:18.016 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:28:18.576 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:28:20.051 [conn46] end connection 127.0.0.4:43813 (4 connections now open) m31000| Wed Jan 16 18:28:20.051 [initandlisten] connection accepted from 127.0.0.4:38409 #47 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1079, "optime" : { "t" : 1358378900000, "i" : 3179 }, "optimeDate" : ISODate("2013-01-16T23:28:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1068, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1068, "optime" : { "t" : 1358378896000, "i" : 1213 }, "optimeDate" : ISODate("2013-01-16T23:28:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1078, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1065, "optime" : { "t" : 1358378899000, "i" : 106 }, "optimeDate" : ISODate("2013-01-16T23:28:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1065, "optime" : { "t" : 1358378900000, "i" : 704 }, "optimeDate" : ISODate("2013-01-16T23:28:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1065, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:19Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1077, "optime" : { "t" : 1358378900000, "i" : 3179 }, "optimeDate" : ISODate("2013-01-16T23:28:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:28:20.576 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:28:22.577 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:28:23.016 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:28:23.722 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:337862 168ms m31000| Wed Jan 16 18:28:23.933 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:266324 133ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1083, "optime" : { "t" : 1358378904000, "i" : 2986 }, "optimeDate" : ISODate("2013-01-16T23:28:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1072, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1072, "optime" : { "t" : 1358378903000, "i" : 211 }, "optimeDate" : ISODate("2013-01-16T23:28:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:24Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1082, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1069, "optime" : { "t" : 1358378900000, "i" : 3179 }, "optimeDate" : ISODate("2013-01-16T23:28:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1069, "optime" : { "t" : 1358378904000, "i" : 398 }, "optimeDate" : ISODate("2013-01-16T23:28:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1069, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1081, "optime" : { "t" : 1358378904000, "i" : 2970 }, "optimeDate" : ISODate("2013-01-16T23:28:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:28:24.577 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:28:26.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:28:26.578 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:28:28.017 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1087, "optime" : { "t" : 1358378908000, "i" : 4366 }, "optimeDate" : ISODate("2013-01-16T23:28:28Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1076, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1076, "optime" : { "t" : 1358378904000, "i" : 2986 }, "optimeDate" : ISODate("2013-01-16T23:28:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:28Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1086, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1073, "optime" : { "t" : 1358378904000, "i" : 2986 }, "optimeDate" : ISODate("2013-01-16T23:28:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:28Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:28Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1073, "optime" : { "t" : 1358378908000, "i" : 823 }, "optimeDate" : ISODate("2013-01-16T23:28:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1073, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1085, "optime" : { "t" : 1358378908000, "i" : 4366 }, "optimeDate" : ISODate("2013-01-16T23:28:28Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:28:28.578 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:28:30.578 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:28:32.579 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1091, "optime" : { "t" : 1358378912000, "i" : 5536 }, "optimeDate" : ISODate("2013-01-16T23:28:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1080, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1080, "optime" : { "t" : 1358378908000, "i" : 4366 }, "optimeDate" : ISODate("2013-01-16T23:28:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:32Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1090, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1077, "optime" : { "t" : 1358378908000, "i" : 4366 }, "optimeDate" : ISODate("2013-01-16T23:28:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1077, "optime" : { "t" : 1358378912000, "i" : 431 }, "optimeDate" : ISODate("2013-01-16T23:28:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1077, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:31Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1089, "optime" : { "t" : 1358378912000, "i" : 5192 }, "optimeDate" : ISODate("2013-01-16T23:28:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:28:33.018 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:28:33.198 [conn67] end connection 127.0.0.3:44526 (3 connections now open) m31002| Wed Jan 16 18:28:33.198 [initandlisten] connection accepted from 127.0.0.3:44222 #69 (4 connections now open) m31000| Wed Jan 16 18:28:34.579 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:28:36.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:28:36.579 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1095, "optime" : { "t" : 1358378916000, "i" : 7217 }, "optimeDate" : ISODate("2013-01-16T23:28:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1084, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1084, "optime" : { "t" : 1358378915000, "i" : 2779 }, "optimeDate" : ISODate("2013-01-16T23:28:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:36Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1094, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1081, "optime" : { "t" : 1358378912000, "i" : 5536 }, "optimeDate" : ISODate("2013-01-16T23:28:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:35Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1081, "optime" : { "t" : 1358378915000, "i" : 2782 }, "optimeDate" : ISODate("2013-01-16T23:28:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1081, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1093, "optime" : { "t" : 1358378916000, "i" : 7170 }, "optimeDate" : ISODate("2013-01-16T23:28:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:28:38.018 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:28:38.580 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:28:40.580 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1099, "optime" : { "t" : 1358378920000, "i" : 8012 }, "optimeDate" : ISODate("2013-01-16T23:28:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1088, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1088, "optime" : { "t" : 1358378916000, "i" : 7217 }, "optimeDate" : ISODate("2013-01-16T23:28:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:40Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1098, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1085, "optime" : { "t" : 1358378916000, "i" : 7217 }, "optimeDate" : ISODate("2013-01-16T23:28:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:40Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1085, "optime" : { "t" : 1358378920000, "i" : 298 }, "optimeDate" : ISODate("2013-01-16T23:28:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1085, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1097, "optime" : { "t" : 1358378920000, "i" : 8012 }, "optimeDate" : ISODate("2013-01-16T23:28:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:28:42.580 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:28:43.019 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:28:44.548 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:301709 150ms m31000| Wed Jan 16 18:28:44.581 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1104, "optime" : { "t" : 1358378925000, "i" : 5 }, "optimeDate" : ISODate("2013-01-16T23:28:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1093, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1093, "optime" : { "t" : 1358378920000, "i" : 8012 }, "optimeDate" : ISODate("2013-01-16T23:28:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:44Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1103, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1090, "optime" : { "t" : 1358378920000, "i" : 8012 }, "optimeDate" : ISODate("2013-01-16T23:28:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1090, "optime" : { "t" : 1358378924000, "i" : 684 }, "optimeDate" : ISODate("2013-01-16T23:28:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1090, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:43Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1102, "optime" : { "t" : 1358378924000, "i" : 9389 }, "optimeDate" : ISODate("2013-01-16T23:28:44Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:28:46.028 [conn68] end connection 127.0.0.2:34988 (3 connections now open) m31002| Wed Jan 16 18:28:46.528 [initandlisten] connection accepted from 127.0.0.2:40905 #70 (4 connections now open) m31001| Wed Jan 16 18:28:46.581 [conn67] end connection 127.0.0.2:44139 (2 connections now open) m31001| Wed Jan 16 18:28:46.581 [initandlisten] connection accepted from 127.0.0.2:54209 #68 (3 connections now open) m31000| Wed Jan 16 18:28:46.581 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:28:48.020 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:28:48.582 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1108, "optime" : { "t" : 1358378929000, "i" : 2156 }, "optimeDate" : ISODate("2013-01-16T23:28:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1097, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1097, "optime" : { "t" : 1358378929000, "i" : 170 }, "optimeDate" : ISODate("2013-01-16T23:28:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1107, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1094, "optime" : { "t" : 1358378929000, "i" : 1650 }, "optimeDate" : ISODate("2013-01-16T23:28:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1094, "optime" : { "t" : 1358378925000, "i" : 5 }, "optimeDate" : ISODate("2013-01-16T23:28:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:49Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1094, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1106, "optime" : { "t" : 1358378929000, "i" : 2156 }, "optimeDate" : ISODate("2013-01-16T23:28:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:28:50.056 [conn47] end connection 127.0.0.4:38409 (4 connections now open) m31000| Wed Jan 16 18:28:50.056 [initandlisten] connection accepted from 127.0.0.4:54797 #48 (5 connections now open) m31001| Wed Jan 16 18:28:50.520 [conn66] end connection 127.0.0.4:42900 (2 connections now open) m31000| Wed Jan 16 18:28:50.582 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:28:51.020 [initandlisten] connection accepted from 127.0.0.4:36826 #69 (3 connections now open) m31000| Wed Jan 16 18:28:52.582 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1112, "optime" : { "t" : 1358378933000, "i" : 4420 }, "optimeDate" : ISODate("2013-01-16T23:28:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1101, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1101, "optime" : { "t" : 1358378929000, "i" : 2156 }, "optimeDate" : ISODate("2013-01-16T23:28:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:52Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1111, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1098, "optime" : { "t" : 1358378933000, "i" : 1979 }, "optimeDate" : ISODate("2013-01-16T23:28:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1098, "optime" : { "t" : 1358378929000, "i" : 2156 }, "optimeDate" : ISODate("2013-01-16T23:28:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1098, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:53Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1110, "optime" : { "t" : 1358378933000, "i" : 4095 }, "optimeDate" : ISODate("2013-01-16T23:28:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:28:53.521 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:28:53.624 [journal] old journal file will be removed: /data/db/testReplSet-0/journal/j._0 m31000| Wed Jan 16 18:28:53.743 [slaveTracking] update local.slaves query: { _id: ObjectId('50f7337af734792a26cd534d'), host: "127.0.0.3", ns: "local.oplog.rs" } update: { $set: { syncedTo: Timestamp 1358378161000|8716 } } nscanned:1 nupdated:1 fastmod:1 keyUpdates:0 locks(micros) w:179 162ms m31000| Wed Jan 16 18:28:54.583 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:28:56.583 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1116, "optime" : { "t" : 1358378937000, "i" : 5570 }, "optimeDate" : ISODate("2013-01-16T23:28:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1105, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1105, "optime" : { "t" : 1358378936000, "i" : 313 }, "optimeDate" : ISODate("2013-01-16T23:28:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1115, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1102, "optime" : { "t" : 1358378937000, "i" : 1270 }, "optimeDate" : ISODate("2013-01-16T23:28:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:28:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1102, "optime" : { "t" : 1358378933000, "i" : 4420 }, "optimeDate" : ISODate("2013-01-16T23:28:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1102, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:28:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1114, "optime" : { "t" : 1358378937000, "i" : 4914 }, "optimeDate" : ISODate("2013-01-16T23:28:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:28:58.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:28:58.058 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:28:58.583 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:29:00.584 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1120, "optime" : { "t" : 1358378941000, "i" : 6899 }, "optimeDate" : ISODate("2013-01-16T23:29:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1109, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1109, "optime" : { "t" : 1358378941000, "i" : 4555 }, "optimeDate" : ISODate("2013-01-16T23:29:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1119, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1106, "optime" : { "t" : 1358378941000, "i" : 1239 }, "optimeDate" : ISODate("2013-01-16T23:29:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1106, "optime" : { "t" : 1358378937000, "i" : 5570 }, "optimeDate" : ISODate("2013-01-16T23:28:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:01Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1106, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1118, "optime" : { "t" : 1358378941000, "i" : 6392 }, "optimeDate" : ISODate("2013-01-16T23:29:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:29:02.584 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:29:03.204 [conn69] end connection 127.0.0.3:44222 (3 connections now open) m31002| Wed Jan 16 18:29:03.204 [initandlisten] connection accepted from 127.0.0.3:40905 #71 (4 connections now open) m31002| Wed Jan 16 18:29:03.522 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:29:04.584 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1124, "optime" : { "t" : 1358378945000, "i" : 7513 }, "optimeDate" : ISODate("2013-01-16T23:29:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1113, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1113, "optime" : { "t" : 1358378941000, "i" : 6899 }, "optimeDate" : ISODate("2013-01-16T23:29:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:04Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1123, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1110, "optime" : { "t" : 1358378945000, "i" : 1314 }, "optimeDate" : ISODate("2013-01-16T23:29:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1110, "optime" : { "t" : 1358378941000, "i" : 6899 }, "optimeDate" : ISODate("2013-01-16T23:29:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1110, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:05Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1122, "optime" : { "t" : 1358378945000, "i" : 7513 }, "optimeDate" : ISODate("2013-01-16T23:29:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:29:06.585 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:29:08.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:29:08.060 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:29:08.585 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1129, "optime" : { "t" : 1358378950000, "i" : 686 }, "optimeDate" : ISODate("2013-01-16T23:29:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1118, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1118, "optime" : { "t" : 1358378949000, "i" : 200 }, "optimeDate" : ISODate("2013-01-16T23:29:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1128, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1115, "optime" : { "t" : 1358378949000, "i" : 1509 }, "optimeDate" : ISODate("2013-01-16T23:29:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1115, "optime" : { "t" : 1358378945000, "i" : 7513 }, "optimeDate" : ISODate("2013-01-16T23:29:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:09Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1115, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:09Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1127, "optime" : { "t" : 1358378950000, "i" : 686 }, "optimeDate" : ISODate("2013-01-16T23:29:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:29:10.586 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:29:12.586 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:29:13.523 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:29:14.189 [journal] old journal file will be removed: /data/db/testReplSet-2/journal/j._0 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1133, "optime" : { "t" : 1358378954000, "i" : 2079 }, "optimeDate" : ISODate("2013-01-16T23:29:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1122, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1122, "optime" : { "t" : 1358378953000, "i" : 7617 }, "optimeDate" : ISODate("2013-01-16T23:29:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:14Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:28:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1132, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1119, "optime" : { "t" : 1358378953000, "i" : 1371 }, "optimeDate" : ISODate("2013-01-16T23:29:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:13Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1119, "optime" : { "t" : 1358378954000, "i" : 682 }, "optimeDate" : ISODate("2013-01-16T23:29:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1119, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1131, "optime" : { "t" : 1358378954000, "i" : 885 }, "optimeDate" : ISODate("2013-01-16T23:29:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:29:14.587 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:29:16.587 [conn68] end connection 127.0.0.2:54209 (2 connections now open) m31001| Wed Jan 16 18:29:16.587 [initandlisten] connection accepted from 127.0.0.2:37286 #70 (3 connections now open) m31000| Wed Jan 16 18:29:16.587 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:29:18.061 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1137, "optime" : { "t" : 1358378958000, "i" : 4004 }, "optimeDate" : ISODate("2013-01-16T23:29:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1126, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1126, "optime" : { "t" : 1358378954000, "i" : 2079 }, "optimeDate" : ISODate("2013-01-16T23:29:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:18Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1136, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1123, "optime" : { "t" : 1358378954000, "i" : 2079 }, "optimeDate" : ISODate("2013-01-16T23:29:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1123, "optime" : { "t" : 1358378958000, "i" : 1011 }, "optimeDate" : ISODate("2013-01-16T23:29:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1123, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1135, "optime" : { "t" : 1358378958000, "i" : 4004 }, "optimeDate" : ISODate("2013-01-16T23:29:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:29:18.588 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:29:20.061 [conn48] end connection 127.0.0.4:54797 (4 connections now open) m31000| Wed Jan 16 18:29:20.062 [initandlisten] connection accepted from 127.0.0.4:57403 #49 (5 connections now open) m31000| Wed Jan 16 18:29:20.588 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1141, "optime" : { "t" : 1358378962000, "i" : 4040 }, "optimeDate" : ISODate("2013-01-16T23:29:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1130, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1130, "optime" : { "t" : 1358378961000, "i" : 1247 }, "optimeDate" : ISODate("2013-01-16T23:29:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:22Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1140, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1127, "optime" : { "t" : 1358378958000, "i" : 4004 }, "optimeDate" : ISODate("2013-01-16T23:29:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1127, "optime" : { "t" : 1358378962000, "i" : 58 }, "optimeDate" : ISODate("2013-01-16T23:29:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1127, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:21Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1139, "optime" : { "t" : 1358378962000, "i" : 4006 }, "optimeDate" : ISODate("2013-01-16T23:29:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:29:22.588 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:29:23.525 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:29:24.033 [conn70] end connection 127.0.0.2:40905 (3 connections now open) m31002| Wed Jan 16 18:29:24.533 [initandlisten] connection accepted from 127.0.0.2:43814 #72 (4 connections now open) m31000| Wed Jan 16 18:29:24.589 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1145, "optime" : { "t" : 1358378966000, "i" : 5313 }, "optimeDate" : ISODate("2013-01-16T23:29:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1134, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1134, "optime" : { "t" : 1358378962000, "i" : 4040 }, "optimeDate" : ISODate("2013-01-16T23:29:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:26Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1144, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1131, "optime" : { "t" : 1358378962000, "i" : 4040 }, "optimeDate" : ISODate("2013-01-16T23:29:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:26Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1131, "optime" : { "t" : 1358378966000, "i" : 1116 }, "optimeDate" : ISODate("2013-01-16T23:29:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1131, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1143, "optime" : { "t" : 1358378966000, "i" : 4956 }, "optimeDate" : ISODate("2013-01-16T23:29:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:29:26.589 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:29:28.063 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:29:28.525 [conn69] end connection 127.0.0.4:36826 (2 connections now open) m31000| Wed Jan 16 18:29:28.590 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:29:29.025 [initandlisten] connection accepted from 127.0.0.4:38294 #71 (3 connections now open) m31001| Wed Jan 16 18:29:30.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:29:30.590 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1149, "optime" : { "t" : 1358378970000, "i" : 6103 }, "optimeDate" : ISODate("2013-01-16T23:29:30Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1138, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1138, "optime" : { "t" : 1358378966000, "i" : 5313 }, "optimeDate" : ISODate("2013-01-16T23:29:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:30Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1148, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1135, "optime" : { "t" : 1358378966000, "i" : 5313 }, "optimeDate" : ISODate("2013-01-16T23:29:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1135, "optime" : { "t" : 1358378970000, "i" : 407 }, "optimeDate" : ISODate("2013-01-16T23:29:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1135, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:29Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1147, "optime" : { "t" : 1358378970000, "i" : 6103 }, "optimeDate" : ISODate("2013-01-16T23:29:30Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:29:32.590 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:29:33.209 [conn71] end connection 127.0.0.3:40905 (3 connections now open) m31002| Wed Jan 16 18:29:33.210 [initandlisten] connection accepted from 127.0.0.3:32770 #73 (4 connections now open) m31002| Wed Jan 16 18:29:34.026 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:29:34.516 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:296450 148ms m31000| Wed Jan 16 18:29:34.591 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1153, "optime" : { "t" : 1358378974000, "i" : 7837 }, "optimeDate" : ISODate("2013-01-16T23:29:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1142, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1142, "optime" : { "t" : 1358378974000, "i" : 3108 }, "optimeDate" : ISODate("2013-01-16T23:29:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1152, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1139, "optime" : { "t" : 1358378970000, "i" : 6103 }, "optimeDate" : ISODate("2013-01-16T23:29:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:34Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1139, "optime" : { "t" : 1358378974000, "i" : 664 }, "optimeDate" : ISODate("2013-01-16T23:29:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1139, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1151, "optime" : { "t" : 1358378974000, "i" : 7837 }, "optimeDate" : ISODate("2013-01-16T23:29:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:29:36.591 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:29:38.591 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1158, "optime" : { "t" : 1358378979000, "i" : 247 }, "optimeDate" : ISODate("2013-01-16T23:29:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1147, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1147, "optime" : { "t" : 1358378974000, "i" : 7837 }, "optimeDate" : ISODate("2013-01-16T23:29:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:38Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1157, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1144, "optime" : { "t" : 1358378974000, "i" : 7837 }, "optimeDate" : ISODate("2013-01-16T23:29:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1144, "optime" : { "t" : 1358378978000, "i" : 584 }, "optimeDate" : ISODate("2013-01-16T23:29:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1144, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1156, "optime" : { "t" : 1358378979000, "i" : 203 }, "optimeDate" : ISODate("2013-01-16T23:29:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:29:39.027 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:29:40.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:29:40.591 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:29:42.592 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1162, "optime" : { "t" : 1358378983000, "i" : 2036 }, "optimeDate" : ISODate("2013-01-16T23:29:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1151, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1151, "optime" : { "t" : 1358378982000, "i" : 106 }, "optimeDate" : ISODate("2013-01-16T23:29:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:42Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1161, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1148, "optime" : { "t" : 1358378979000, "i" : 247 }, "optimeDate" : ISODate("2013-01-16T23:29:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:41Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1148, "optime" : { "t" : 1358378982000, "i" : 219 }, "optimeDate" : ISODate("2013-01-16T23:29:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1148, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1160, "optime" : { "t" : 1358378983000, "i" : 2036 }, "optimeDate" : ISODate("2013-01-16T23:29:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:29:44.028 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:29:44.592 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:29:46.592 [conn70] end connection 127.0.0.2:37286 (2 connections now open) m31001| Wed Jan 16 18:29:46.593 [initandlisten] connection accepted from 127.0.0.2:45569 #72 (3 connections now open) m31000| Wed Jan 16 18:29:46.593 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1166, "optime" : { "t" : 1358378987000, "i" : 2943 }, "optimeDate" : ISODate("2013-01-16T23:29:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1155, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1155, "optime" : { "t" : 1358378986000, "i" : 6887 }, "optimeDate" : ISODate("2013-01-16T23:29:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1165, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1152, "optime" : { "t" : 1358378987000, "i" : 1596 }, "optimeDate" : ISODate("2013-01-16T23:29:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1152, "optime" : { "t" : 1358378983000, "i" : 2036 }, "optimeDate" : ISODate("2013-01-16T23:29:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:47Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1152, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1164, "optime" : { "t" : 1358378987000, "i" : 2943 }, "optimeDate" : ISODate("2013-01-16T23:29:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:29:48.593 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:29:49.029 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:29:50.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:29:50.066 [conn49] end connection 127.0.0.4:57403 (4 connections now open) m31000| Wed Jan 16 18:29:50.066 [initandlisten] connection accepted from 127.0.0.4:33062 #50 (5 connections now open) m31000| Wed Jan 16 18:29:50.593 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1170, "optime" : { "t" : 1358378991000, "i" : 4655 }, "optimeDate" : ISODate("2013-01-16T23:29:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1159, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1159, "optime" : { "t" : 1358378987000, "i" : 2943 }, "optimeDate" : ISODate("2013-01-16T23:29:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:50Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1169, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1156, "optime" : { "t" : 1358378991000, "i" : 1758 }, "optimeDate" : ISODate("2013-01-16T23:29:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:51Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1156, "optime" : { "t" : 1358378987000, "i" : 2943 }, "optimeDate" : ISODate("2013-01-16T23:29:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1156, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1168, "optime" : { "t" : 1358378991000, "i" : 4648 }, "optimeDate" : ISODate("2013-01-16T23:29:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:29:52.594 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:29:54.029 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:29:54.594 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1174, "optime" : { "t" : 1358378995000, "i" : 7047 }, "optimeDate" : ISODate("2013-01-16T23:29:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1163, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1163, "optime" : { "t" : 1358378991000, "i" : 4655 }, "optimeDate" : ISODate("2013-01-16T23:29:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1173, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1160, "optime" : { "t" : 1358378995000, "i" : 2198 }, "optimeDate" : ISODate("2013-01-16T23:29:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1160, "optime" : { "t" : 1358378991000, "i" : 4655 }, "optimeDate" : ISODate("2013-01-16T23:29:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1160, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1172, "optime" : { "t" : 1358378995000, "i" : 7047 }, "optimeDate" : ISODate("2013-01-16T23:29:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:29:56.595 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:29:58.595 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:29:59.030 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:29:59.551 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:243934 121ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1178, "optime" : { "t" : 1358378999000, "i" : 8139 }, "optimeDate" : ISODate("2013-01-16T23:29:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1167, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1167, "optime" : { "t" : 1358378999000, "i" : 3869 }, "optimeDate" : ISODate("2013-01-16T23:29:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1177, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1164, "optime" : { "t" : 1358378999000, "i" : 2000 }, "optimeDate" : ISODate("2013-01-16T23:29:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:29:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1164, "optime" : { "t" : 1358378995000, "i" : 7047 }, "optimeDate" : ISODate("2013-01-16T23:29:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:59Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1164, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:29:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:29:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1176, "optime" : { "t" : 1358378999000, "i" : 8139 }, "optimeDate" : ISODate("2013-01-16T23:29:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:30:00.595 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:30:02.038 [conn72] end connection 127.0.0.2:43814 (3 connections now open) m31002| Wed Jan 16 18:30:02.538 [initandlisten] connection accepted from 127.0.0.2:60822 #74 (4 connections now open) m31000| Wed Jan 16 18:30:02.596 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:30:03.215 [conn73] end connection 127.0.0.3:32770 (3 connections now open) m31002| Wed Jan 16 18:30:03.215 [initandlisten] connection accepted from 127.0.0.3:51556 #75 (4 connections now open) m31002| Wed Jan 16 18:30:04.030 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1183, "optime" : { "t" : 1358379004000, "i" : 754 }, "optimeDate" : ISODate("2013-01-16T23:30:04Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1172, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1172, "optime" : { "t" : 1358378999000, "i" : 8139 }, "optimeDate" : ISODate("2013-01-16T23:29:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:04Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1182, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1169, "optime" : { "t" : 1358379003000, "i" : 1675 }, "optimeDate" : ISODate("2013-01-16T23:30:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:04Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1169, "optime" : { "t" : 1358379004000, "i" : 723 }, "optimeDate" : ISODate("2013-01-16T23:30:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1169, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1181, "optime" : { "t" : 1358379004000, "i" : 754 }, "optimeDate" : ISODate("2013-01-16T23:30:04Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:30:04.596 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:30:06.531 [conn71] end connection 127.0.0.4:38294 (2 connections now open) m31000| Wed Jan 16 18:30:06.597 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:30:07.031 [initandlisten] connection accepted from 127.0.0.4:34866 #73 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1187, "optime" : { "t" : 1358379008000, "i" : 2089 }, "optimeDate" : ISODate("2013-01-16T23:30:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1176, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1176, "optime" : { "t" : 1358379007000, "i" : 4140 }, "optimeDate" : ISODate("2013-01-16T23:30:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:08Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1186, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1173, "optime" : { "t" : 1358379007000, "i" : 726 }, "optimeDate" : ISODate("2013-01-16T23:30:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1173, "optime" : { "t" : 1358379008000, "i" : 392 }, "optimeDate" : ISODate("2013-01-16T23:30:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1173, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:07Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1185, "optime" : { "t" : 1358379008000, "i" : 2089 }, "optimeDate" : ISODate("2013-01-16T23:30:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:30:08.597 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:30:09.532 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:30:10.597 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:30:12.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1191, "optime" : { "t" : 1358379012000, "i" : 3415 }, "optimeDate" : ISODate("2013-01-16T23:30:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1180, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1180, "optime" : { "t" : 1358379008000, "i" : 2089 }, "optimeDate" : ISODate("2013-01-16T23:30:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:12Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1190, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1177, "optime" : { "t" : 1358379008000, "i" : 2089 }, "optimeDate" : ISODate("2013-01-16T23:30:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:12Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1177, "optime" : { "t" : 1358379012000, "i" : 1140 }, "optimeDate" : ISODate("2013-01-16T23:30:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1177, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1189, "optime" : { "t" : 1358379012000, "i" : 3310 }, "optimeDate" : ISODate("2013-01-16T23:30:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:30:12.598 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:30:14.070 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:30:14.598 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1195, "optime" : { "t" : 1358379016000, "i" : 4380 }, "optimeDate" : ISODate("2013-01-16T23:30:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1184, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1184, "optime" : { "t" : 1358379012000, "i" : 3415 }, "optimeDate" : ISODate("2013-01-16T23:30:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1194, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1181, "optime" : { "t" : 1358379012000, "i" : 3415 }, "optimeDate" : ISODate("2013-01-16T23:30:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1181, "optime" : { "t" : 1358379016000, "i" : 64 }, "optimeDate" : ISODate("2013-01-16T23:30:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1181, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:15Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1193, "optime" : { "t" : 1358379016000, "i" : 4380 }, "optimeDate" : ISODate("2013-01-16T23:30:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:30:16.599 [conn72] end connection 127.0.0.2:45569 (2 connections now open) m31001| Wed Jan 16 18:30:16.599 [initandlisten] connection accepted from 127.0.0.2:57574 #74 (3 connections now open) m31000| Wed Jan 16 18:30:16.599 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:30:18.599 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:30:19.533 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:30:20.071 [conn50] end connection 127.0.0.4:33062 (4 connections now open) m31000| Wed Jan 16 18:30:20.071 [initandlisten] connection accepted from 127.0.0.4:43408 #51 (5 connections now open) m31000| Wed Jan 16 18:30:20.600 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1199, "optime" : { "t" : 1358379020000, "i" : 6112 }, "optimeDate" : ISODate("2013-01-16T23:30:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1188, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1188, "optime" : { "t" : 1358379020000, "i" : 279 }, "optimeDate" : ISODate("2013-01-16T23:30:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1198, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1185, "optime" : { "t" : 1358379016000, "i" : 4380 }, "optimeDate" : ISODate("2013-01-16T23:30:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:19Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1185, "optime" : { "t" : 1358379020000, "i" : 522 }, "optimeDate" : ISODate("2013-01-16T23:30:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1185, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1197, "optime" : { "t" : 1358379020000, "i" : 6011 }, "optimeDate" : ISODate("2013-01-16T23:30:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:30:22.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:30:22.600 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:30:24.072 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:30:24.600 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1203, "optime" : { "t" : 1358379024000, "i" : 7498 }, "optimeDate" : ISODate("2013-01-16T23:30:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1192, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1192, "optime" : { "t" : 1358379020000, "i" : 6112 }, "optimeDate" : ISODate("2013-01-16T23:30:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:24Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1202, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1189, "optime" : { "t" : 1358379020000, "i" : 6112 }, "optimeDate" : ISODate("2013-01-16T23:30:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:24Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1189, "optime" : { "t" : 1358379024000, "i" : 655 }, "optimeDate" : ISODate("2013-01-16T23:30:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1189, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1201, "optime" : { "t" : 1358379024000, "i" : 7498 }, "optimeDate" : ISODate("2013-01-16T23:30:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:30:26.601 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:30:28.601 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1208, "optime" : { "t" : 1358379029000, "i" : 473 }, "optimeDate" : ISODate("2013-01-16T23:30:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1197, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1197, "optime" : { "t" : 1358379024000, "i" : 7498 }, "optimeDate" : ISODate("2013-01-16T23:30:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:28Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1207, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1194, "optime" : { "t" : 1358379024000, "i" : 7498 }, "optimeDate" : ISODate("2013-01-16T23:30:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1194, "optime" : { "t" : 1358379028000, "i" : 839 }, "optimeDate" : ISODate("2013-01-16T23:30:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1194, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:27Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1206, "optime" : { "t" : 1358379029000, "i" : 473 }, "optimeDate" : ISODate("2013-01-16T23:30:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:30:29.534 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:30:30.601 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:30:32.602 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1212, "optime" : { "t" : 1358379033000, "i" : 1451 }, "optimeDate" : ISODate("2013-01-16T23:30:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1201, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1201, "optime" : { "t" : 1358379032000, "i" : 4591 }, "optimeDate" : ISODate("2013-01-16T23:30:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1211, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1198, "optime" : { "t" : 1358379029000, "i" : 473 }, "optimeDate" : ISODate("2013-01-16T23:30:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:32Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1198, "optime" : { "t" : 1358379032000, "i" : 112 }, "optimeDate" : ISODate("2013-01-16T23:30:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1198, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1210, "optime" : { "t" : 1358379033000, "i" : 1420 }, "optimeDate" : ISODate("2013-01-16T23:30:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:30:33.220 [conn75] end connection 127.0.0.3:51556 (3 connections now open) m31002| Wed Jan 16 18:30:33.220 [initandlisten] connection accepted from 127.0.0.3:44618 #76 (4 connections now open) m31002| Wed Jan 16 18:30:34.074 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:30:34.602 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:30:36.602 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1216, "optime" : { "t" : 1358379037000, "i" : 3047 }, "optimeDate" : ISODate("2013-01-16T23:30:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1205, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1205, "optime" : { "t" : 1358379033000, "i" : 1451 }, "optimeDate" : ISODate("2013-01-16T23:30:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:36Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1215, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1202, "optime" : { "t" : 1358379037000, "i" : 1974 }, "optimeDate" : ISODate("2013-01-16T23:30:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1202, "optime" : { "t" : 1358379033000, "i" : 1451 }, "optimeDate" : ISODate("2013-01-16T23:30:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1202, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1214, "optime" : { "t" : 1358379037000, "i" : 3047 }, "optimeDate" : ISODate("2013-01-16T23:30:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:30:38.603 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:30:39.535 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:30:40.043 [conn74] end connection 127.0.0.2:60822 (3 connections now open) m31002| Wed Jan 16 18:30:40.543 [initandlisten] connection accepted from 127.0.0.2:46749 #77 (4 connections now open) m31000| Wed Jan 16 18:30:40.603 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1220, "optime" : { "t" : 1358379041000, "i" : 4098 }, "optimeDate" : ISODate("2013-01-16T23:30:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1209, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1209, "optime" : { "t" : 1358379040000, "i" : 1240 }, "optimeDate" : ISODate("2013-01-16T23:30:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1219, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1206, "optime" : { "t" : 1358379041000, "i" : 2082 }, "optimeDate" : ISODate("2013-01-16T23:30:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:41Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1206, "optime" : { "t" : 1358379037000, "i" : 3047 }, "optimeDate" : ISODate("2013-01-16T23:30:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1206, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:41Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1218, "optime" : { "t" : 1358379041000, "i" : 3454 }, "optimeDate" : ISODate("2013-01-16T23:30:41Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:30:42.603 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:30:44.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:30:44.076 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:30:44.536 [conn73] end connection 127.0.0.4:34866 (2 connections now open) m31000| Wed Jan 16 18:30:44.604 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:30:44.633 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:309952 154ms m31001| Wed Jan 16 18:30:45.036 [initandlisten] connection accepted from 127.0.0.4:38398 #75 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1224, "optime" : { "t" : 1358379045000, "i" : 5153 }, "optimeDate" : ISODate("2013-01-16T23:30:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1213, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1213, "optime" : { "t" : 1358379045000, "i" : 4504 }, "optimeDate" : ISODate("2013-01-16T23:30:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1223, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1210, "optime" : { "t" : 1358379045000, "i" : 1625 }, "optimeDate" : ISODate("2013-01-16T23:30:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1210, "optime" : { "t" : 1358379041000, "i" : 4098 }, "optimeDate" : ISODate("2013-01-16T23:30:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:45Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1210, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1222, "optime" : { "t" : 1358379045000, "i" : 4563 }, "optimeDate" : ISODate("2013-01-16T23:30:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:30:46.604 [conn74] end connection 127.0.0.2:57574 (2 connections now open) m31001| Wed Jan 16 18:30:46.604 [initandlisten] connection accepted from 127.0.0.2:48071 #76 (3 connections now open) m31000| Wed Jan 16 18:30:46.604 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:30:48.605 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:30:49.665 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:377031 188ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1228, "optime" : { "t" : 1358379049000, "i" : 6878 }, "optimeDate" : ISODate("2013-01-16T23:30:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1217, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1217, "optime" : { "t" : 1358379045000, "i" : 5153 }, "optimeDate" : ISODate("2013-01-16T23:30:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:48Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1227, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1214, "optime" : { "t" : 1358379049000, "i" : 1508 }, "optimeDate" : ISODate("2013-01-16T23:30:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1214, "optime" : { "t" : 1358379045000, "i" : 5153 }, "optimeDate" : ISODate("2013-01-16T23:30:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1214, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1226, "optime" : { "t" : 1358379049000, "i" : 6795 }, "optimeDate" : ISODate("2013-01-16T23:30:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:30:50.037 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:30:50.077 [conn51] end connection 127.0.0.4:43408 (4 connections now open) m31000| Wed Jan 16 18:30:50.077 [initandlisten] connection accepted from 127.0.0.4:54278 #52 (5 connections now open) m31000| Wed Jan 16 18:30:50.605 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:30:52.605 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1232, "optime" : { "t" : 1358379053000, "i" : 8855 }, "optimeDate" : ISODate("2013-01-16T23:30:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1221, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1221, "optime" : { "t" : 1358379053000, "i" : 330 }, "optimeDate" : ISODate("2013-01-16T23:30:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1231, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1218, "optime" : { "t" : 1358379053000, "i" : 2092 }, "optimeDate" : ISODate("2013-01-16T23:30:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1218, "optime" : { "t" : 1358379049000, "i" : 6878 }, "optimeDate" : ISODate("2013-01-16T23:30:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:53Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1218, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:53Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1230, "optime" : { "t" : 1358379053000, "i" : 8588 }, "optimeDate" : ISODate("2013-01-16T23:30:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:30:54.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:30:54.606 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:30:55.037 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:30:56.606 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1237, "optime" : { "t" : 1358379058000, "i" : 1661 }, "optimeDate" : ISODate("2013-01-16T23:30:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1226, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1226, "optime" : { "t" : 1358379058000, "i" : 58 }, "optimeDate" : ISODate("2013-01-16T23:30:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:58Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1236, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1223, "optime" : { "t" : 1358379057000, "i" : 1864 }, "optimeDate" : ISODate("2013-01-16T23:30:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:30:57Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:30:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1223, "optime" : { "t" : 1358379058000, "i" : 559 }, "optimeDate" : ISODate("2013-01-16T23:30:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1223, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1235, "optime" : { "t" : 1358379058000, "i" : 1661 }, "optimeDate" : ISODate("2013-01-16T23:30:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:30:58.607 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:31:00.038 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:31:00.607 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:31:01.617 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:212143 106ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1241, "optime" : { "t" : 1358379062000, "i" : 2429 }, "optimeDate" : ISODate("2013-01-16T23:31:02Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1230, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1230, "optime" : { "t" : 1358379058000, "i" : 1661 }, "optimeDate" : ISODate("2013-01-16T23:30:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:02Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1240, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1227, "optime" : { "t" : 1358379061000, "i" : 17 }, "optimeDate" : ISODate("2013-01-16T23:31:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1227, "optime" : { "t" : 1358379062000, "i" : 283 }, "optimeDate" : ISODate("2013-01-16T23:31:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1227, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:30:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1239, "optime" : { "t" : 1358379062000, "i" : 1818 }, "optimeDate" : ISODate("2013-01-16T23:31:02Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:31:02.607 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:31:03.225 [conn76] end connection 127.0.0.3:44618 (3 connections now open) m31002| Wed Jan 16 18:31:03.225 [initandlisten] connection accepted from 127.0.0.3:42196 #78 (4 connections now open) m31000| Wed Jan 16 18:31:04.608 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:31:05.039 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1245, "optime" : { "t" : 1358379066000, "i" : 3768 }, "optimeDate" : ISODate("2013-01-16T23:31:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1234, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1234, "optime" : { "t" : 1358379065000, "i" : 1970 }, "optimeDate" : ISODate("2013-01-16T23:31:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:06Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1244, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1231, "optime" : { "t" : 1358379062000, "i" : 2429 }, "optimeDate" : ISODate("2013-01-16T23:31:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1231, "optime" : { "t" : 1358379066000, "i" : 1117 }, "optimeDate" : ISODate("2013-01-16T23:31:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1231, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:05Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1243, "optime" : { "t" : 1358379066000, "i" : 3768 }, "optimeDate" : ISODate("2013-01-16T23:31:06Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:31:06.608 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:31:08.609 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:31:10.039 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1249, "optime" : { "t" : 1358379070000, "i" : 4793 }, "optimeDate" : ISODate("2013-01-16T23:31:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1238, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1238, "optime" : { "t" : 1358379066000, "i" : 3768 }, "optimeDate" : ISODate("2013-01-16T23:31:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:10Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1248, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1235, "optime" : { "t" : 1358379066000, "i" : 3768 }, "optimeDate" : ISODate("2013-01-16T23:31:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:10Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1235, "optime" : { "t" : 1358379070000, "i" : 1286 }, "optimeDate" : ISODate("2013-01-16T23:31:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1235, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1247, "optime" : { "t" : 1358379070000, "i" : 4657 }, "optimeDate" : ISODate("2013-01-16T23:31:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:31:10.609 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:31:12.609 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:31:14.610 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1253, "optime" : { "t" : 1358379074000, "i" : 5389 }, "optimeDate" : ISODate("2013-01-16T23:31:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1242, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1242, "optime" : { "t" : 1358379070000, "i" : 4793 }, "optimeDate" : ISODate("2013-01-16T23:31:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:14Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1252, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1239, "optime" : { "t" : 1358379070000, "i" : 4793 }, "optimeDate" : ISODate("2013-01-16T23:31:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1239, "optime" : { "t" : 1358379074000, "i" : 347 }, "optimeDate" : ISODate("2013-01-16T23:31:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1239, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:13Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1251, "optime" : { "t" : 1358379074000, "i" : 5311 }, "optimeDate" : ISODate("2013-01-16T23:31:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:31:15.040 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:31:16.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 18:31:16.610 [conn76] end connection 127.0.0.2:48071 (2 connections now open) m31001| Wed Jan 16 18:31:16.610 [initandlisten] connection accepted from 127.0.0.2:41003 #77 (3 connections now open) m31000| Wed Jan 16 18:31:16.610 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:31:18.048 [conn77] end connection 127.0.0.2:46749 (3 connections now open) m31002| Wed Jan 16 18:31:18.548 [initandlisten] connection accepted from 127.0.0.2:57000 #79 (4 connections now open) m31000| Wed Jan 16 18:31:18.611 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1257, "optime" : { "t" : 1358379078000, "i" : 6994 }, "optimeDate" : ISODate("2013-01-16T23:31:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1246, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1246, "optime" : { "t" : 1358379078000, "i" : 4194 }, "optimeDate" : ISODate("2013-01-16T23:31:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1256, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1243, "optime" : { "t" : 1358379074000, "i" : 5389 }, "optimeDate" : ISODate("2013-01-16T23:31:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:17Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1243, "optime" : { "t" : 1358379078000, "i" : 308 }, "optimeDate" : ISODate("2013-01-16T23:31:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1243, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1255, "optime" : { "t" : 1358379078000, "i" : 6994 }, "optimeDate" : ISODate("2013-01-16T23:31:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:31:20.041 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:31:20.082 [conn52] end connection 127.0.0.4:54278 (4 connections now open) m31000| Wed Jan 16 18:31:20.082 [initandlisten] connection accepted from 127.0.0.4:37585 #53 (5 connections now open) m31000| Wed Jan 16 18:31:20.611 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:31:22.075 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:251887 125ms m31001| Wed Jan 16 18:31:22.541 [conn75] end connection 127.0.0.4:38398 (2 connections now open) m31000| Wed Jan 16 18:31:22.611 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 m31001| Wed Jan 16 18:31:23.041 [initandlisten] connection accepted from 127.0.0.4:37748 #78 (3 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1262, "optime" : { "t" : 1358379083000, "i" : 70 }, "optimeDate" : ISODate("2013-01-16T23:31:23Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1251, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1251, "optime" : { "t" : 1358379078000, "i" : 6994 }, "optimeDate" : ISODate("2013-01-16T23:31:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:22Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1261, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1248, "optime" : { "t" : 1358379078000, "i" : 6994 }, "optimeDate" : ISODate("2013-01-16T23:31:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:23Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1248, "optime" : { "t" : 1358379082000, "i" : 276 }, "optimeDate" : ISODate("2013-01-16T23:31:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1248, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1260, "optime" : { "t" : 1358379083000, "i" : 41 }, "optimeDate" : ISODate("2013-01-16T23:31:23Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:31:24.612 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:31:25.542 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:31:26.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:31:26.612 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1266, "optime" : { "t" : 1358379087000, "i" : 2557 }, "optimeDate" : ISODate("2013-01-16T23:31:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1255, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1255, "optime" : { "t" : 1358379083000, "i" : 70 }, "optimeDate" : ISODate("2013-01-16T23:31:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:26Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1265, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1252, "optime" : { "t" : 1358379087000, "i" : 2431 }, "optimeDate" : ISODate("2013-01-16T23:31:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1252, "optime" : { "t" : 1358379083000, "i" : 70 }, "optimeDate" : ISODate("2013-01-16T23:31:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1252, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:27Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1264, "optime" : { "t" : 1358379087000, "i" : 2495 }, "optimeDate" : ISODate("2013-01-16T23:31:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:31:28.612 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:31:30.084 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:31:30.613 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1270, "optime" : { "t" : 1358379091000, "i" : 4171 }, "optimeDate" : ISODate("2013-01-16T23:31:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1259, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1259, "optime" : { "t" : 1358379091000, "i" : 813 }, "optimeDate" : ISODate("2013-01-16T23:31:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1269, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1256, "optime" : { "t" : 1358379091000, "i" : 2370 }, "optimeDate" : ISODate("2013-01-16T23:31:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1256, "optime" : { "t" : 1358379087000, "i" : 2557 }, "optimeDate" : ISODate("2013-01-16T23:31:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:31Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1256, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:31Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1268, "optime" : { "t" : 1358379091000, "i" : 4171 }, "optimeDate" : ISODate("2013-01-16T23:31:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:31:32.613 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:31:33.231 [conn78] end connection 127.0.0.3:42196 (3 connections now open) m31002| Wed Jan 16 18:31:33.231 [initandlisten] connection accepted from 127.0.0.3:43039 #80 (4 connections now open) m31000| Wed Jan 16 18:31:34.613 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:31:35.543 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1274, "optime" : { "t" : 1358379095000, "i" : 6310 }, "optimeDate" : ISODate("2013-01-16T23:31:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1263, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1263, "optime" : { "t" : 1358379091000, "i" : 4171 }, "optimeDate" : ISODate("2013-01-16T23:31:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:34Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1273, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1260, "optime" : { "t" : 1358379095000, "i" : 2419 }, "optimeDate" : ISODate("2013-01-16T23:31:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:35Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1260, "optime" : { "t" : 1358379091000, "i" : 4171 }, "optimeDate" : ISODate("2013-01-16T23:31:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1260, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1272, "optime" : { "t" : 1358379095000, "i" : 6213 }, "optimeDate" : ISODate("2013-01-16T23:31:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:31:36.614 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:31:38.614 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1278, "optime" : { "t" : 1358379099000, "i" : 8340 }, "optimeDate" : ISODate("2013-01-16T23:31:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1267, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1267, "optime" : { "t" : 1358379095000, "i" : 6310 }, "optimeDate" : ISODate("2013-01-16T23:31:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1277, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1264, "optime" : { "t" : 1358379099000, "i" : 2149 }, "optimeDate" : ISODate("2013-01-16T23:31:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1264, "optime" : { "t" : 1358379095000, "i" : 6310 }, "optimeDate" : ISODate("2013-01-16T23:31:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1264, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:39Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1276, "optime" : { "t" : 1358379099000, "i" : 8340 }, "optimeDate" : ISODate("2013-01-16T23:31:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:31:40.085 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:31:40.614 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:31:42.615 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1283, "optime" : { "t" : 1358379104000, "i" : 372 }, "optimeDate" : ISODate("2013-01-16T23:31:44Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1272, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1272, "optime" : { "t" : 1358379103000, "i" : 5040 }, "optimeDate" : ISODate("2013-01-16T23:31:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1282, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1269, "optime" : { "t" : 1358379103000, "i" : 2186 }, "optimeDate" : ISODate("2013-01-16T23:31:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1269, "optime" : { "t" : 1358379099000, "i" : 8340 }, "optimeDate" : ISODate("2013-01-16T23:31:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:43Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1269, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:43Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1281, "optime" : { "t" : 1358379103000, "i" : 9143 }, "optimeDate" : ISODate("2013-01-16T23:31:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:31:44.615 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:31:45.544 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:31:46.616 [conn77] end connection 127.0.0.2:41003 (2 connections now open) m31001| Wed Jan 16 18:31:46.616 [initandlisten] connection accepted from 127.0.0.2:60897 #79 (3 connections now open) m31000| Wed Jan 16 18:31:46.616 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:31:48.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1287, "optime" : { "t" : 1358379108000, "i" : 1750 }, "optimeDate" : ISODate("2013-01-16T23:31:48Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1276, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1276, "optime" : { "t" : 1358379104000, "i" : 372 }, "optimeDate" : ISODate("2013-01-16T23:31:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:48Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1286, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1273, "optime" : { "t" : 1358379107000, "i" : 1741 }, "optimeDate" : ISODate("2013-01-16T23:31:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:48Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1273, "optime" : { "t" : 1358379108000, "i" : 848 }, "optimeDate" : ISODate("2013-01-16T23:31:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1273, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1285, "optime" : { "t" : 1358379108000, "i" : 1750 }, "optimeDate" : ISODate("2013-01-16T23:31:48Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:31:48.616 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:31:50.087 [conn53] end connection 127.0.0.4:37585 (4 connections now open) m31000| Wed Jan 16 18:31:50.087 [initandlisten] connection accepted from 127.0.0.4:38886 #54 (5 connections now open) m31002| Wed Jan 16 18:31:50.087 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:31:50.617 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1291, "optime" : { "t" : 1358379112000, "i" : 2769 }, "optimeDate" : ISODate("2013-01-16T23:31:52Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1280, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1280, "optime" : { "t" : 1358379108000, "i" : 1750 }, "optimeDate" : ISODate("2013-01-16T23:31:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:52Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1290, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1277, "optime" : { "t" : 1358379111000, "i" : 15 }, "optimeDate" : ISODate("2013-01-16T23:31:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1277, "optime" : { "t" : 1358379112000, "i" : 531 }, "optimeDate" : ISODate("2013-01-16T23:31:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1277, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1289, "optime" : { "t" : 1358379112000, "i" : 2113 }, "optimeDate" : ISODate("2013-01-16T23:31:52Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:31:52.617 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:31:54.618 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:31:55.545 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:31:56.053 [conn79] end connection 127.0.0.2:57000 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1295, "optime" : { "t" : 1358379116000, "i" : 3569 }, "optimeDate" : ISODate("2013-01-16T23:31:56Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1284, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1284, "optime" : { "t" : 1358379112000, "i" : 2769 }, "optimeDate" : ISODate("2013-01-16T23:31:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:56Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1294, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1281, "optime" : { "t" : 1358379112000, "i" : 2769 }, "optimeDate" : ISODate("2013-01-16T23:31:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:55Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:31:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1281, "optime" : { "t" : 1358379116000, "i" : 356 }, "optimeDate" : ISODate("2013-01-16T23:31:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1281, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1293, "optime" : { "t" : 1358379116000, "i" : 3199 }, "optimeDate" : ISODate("2013-01-16T23:31:56Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:31:56.553 [initandlisten] connection accepted from 127.0.0.2:39181 #81 (4 connections now open) m31000| Wed Jan 16 18:31:56.618 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:31:58.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:31:58.618 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:32:00.089 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:32:00.546 [conn78] end connection 127.0.0.4:37748 (2 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1299, "optime" : { "t" : 1358379120000, "i" : 5188 }, "optimeDate" : ISODate("2013-01-16T23:32:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1288, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1288, "optime" : { "t" : 1358379116000, "i" : 3569 }, "optimeDate" : ISODate("2013-01-16T23:31:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1298, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1285, "optime" : { "t" : 1358379116000, "i" : 3569 }, "optimeDate" : ISODate("2013-01-16T23:31:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1285, "optime" : { "t" : 1358379120000, "i" : 1045 }, "optimeDate" : ISODate("2013-01-16T23:32:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1285, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:31:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1297, "optime" : { "t" : 1358379120000, "i" : 5188 }, "optimeDate" : ISODate("2013-01-16T23:32:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:32:00.619 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:32:01.046 [initandlisten] connection accepted from 127.0.0.4:42861 #80 (3 connections now open) m31000| Wed Jan 16 18:32:02.619 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:32:03.237 [conn80] end connection 127.0.0.3:43039 (3 connections now open) m31002| Wed Jan 16 18:32:03.237 [initandlisten] connection accepted from 127.0.0.3:56193 #82 (4 connections now open) m31000| Wed Jan 16 18:32:04.619 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1303, "optime" : { "t" : 1358379124000, "i" : 6483 }, "optimeDate" : ISODate("2013-01-16T23:32:04Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1292, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1292, "optime" : { "t" : 1358379124000, "i" : 651 }, "optimeDate" : ISODate("2013-01-16T23:32:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:04Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:31:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1302, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1289, "optime" : { "t" : 1358379120000, "i" : 5188 }, "optimeDate" : ISODate("2013-01-16T23:32:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:03Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1289, "optime" : { "t" : 1358379124000, "i" : 1189 }, "optimeDate" : ISODate("2013-01-16T23:32:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1289, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1301, "optime" : { "t" : 1358379124000, "i" : 6483 }, "optimeDate" : ISODate("2013-01-16T23:32:04Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:32:06.047 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:32:06.620 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:32:08.620 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1307, "optime" : { "t" : 1358379128000, "i" : 7635 }, "optimeDate" : ISODate("2013-01-16T23:32:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1296, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1296, "optime" : { "t" : 1358379124000, "i" : 6483 }, "optimeDate" : ISODate("2013-01-16T23:32:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:08Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1306, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1293, "optime" : { "t" : 1358379124000, "i" : 6483 }, "optimeDate" : ISODate("2013-01-16T23:32:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:08Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1293, "optime" : { "t" : 1358379128000, "i" : 493 }, "optimeDate" : ISODate("2013-01-16T23:32:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1293, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1305, "optime" : { "t" : 1358379128000, "i" : 7194 }, "optimeDate" : ISODate("2013-01-16T23:32:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:32:10.621 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:32:11.048 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:32:12.621 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1312, "optime" : { "t" : 1358379133000, "i" : 217 }, "optimeDate" : ISODate("2013-01-16T23:32:13Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1301, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1301, "optime" : { "t" : 1358379128000, "i" : 7635 }, "optimeDate" : ISODate("2013-01-16T23:32:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:12Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1311, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1298, "optime" : { "t" : 1358379128000, "i" : 7635 }, "optimeDate" : ISODate("2013-01-16T23:32:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1298, "optime" : { "t" : 1358379132000, "i" : 971 }, "optimeDate" : ISODate("2013-01-16T23:32:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1298, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1310, "optime" : { "t" : 1358379133000, "i" : 217 }, "optimeDate" : ISODate("2013-01-16T23:32:13Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:32:14.621 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:32:16.048 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:32:16.622 [conn79] end connection 127.0.0.2:60897 (2 connections now open) m31001| Wed Jan 16 18:32:16.622 [initandlisten] connection accepted from 127.0.0.2:45204 #81 (3 connections now open) m31000| Wed Jan 16 18:32:16.622 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1316, "optime" : { "t" : 1358379137000, "i" : 1796 }, "optimeDate" : ISODate("2013-01-16T23:32:17Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1305, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1305, "optime" : { "t" : 1358379136000, "i" : 4015 }, "optimeDate" : ISODate("2013-01-16T23:32:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1315, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1302, "optime" : { "t" : 1358379133000, "i" : 217 }, "optimeDate" : ISODate("2013-01-16T23:32:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:16Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1302, "optime" : { "t" : 1358379136000, "i" : 387 }, "optimeDate" : ISODate("2013-01-16T23:32:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1302, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1314, "optime" : { "t" : 1358379137000, "i" : 1771 }, "optimeDate" : ISODate("2013-01-16T23:32:17Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:32:18.622 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:32:20.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:32:20.092 [conn54] end connection 127.0.0.4:38886 (4 connections now open) m31000| Wed Jan 16 18:32:20.092 [initandlisten] connection accepted from 127.0.0.4:55634 #55 (5 connections now open) m31000| Wed Jan 16 18:32:20.378 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:347821 173ms m31000| Wed Jan 16 18:32:20.622 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:32:20.635 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:373449 186ms m31000| Wed Jan 16 18:32:20.894 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:377109 188ms m31002| Wed Jan 16 18:32:21.049 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1320, "optime" : { "t" : 1358379141000, "i" : 2797 }, "optimeDate" : ISODate("2013-01-16T23:32:21Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1309, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1309, "optime" : { "t" : 1358379137000, "i" : 1796 }, "optimeDate" : ISODate("2013-01-16T23:32:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1319, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1306, "optime" : { "t" : 1358379137000, "i" : 1796 }, "optimeDate" : ISODate("2013-01-16T23:32:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:21Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1306, "optime" : { "t" : 1358379137000, "i" : 1796 }, "optimeDate" : ISODate("2013-01-16T23:32:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1306, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1318, "optime" : { "t" : 1358379141000, "i" : 982 }, "optimeDate" : ISODate("2013-01-16T23:32:21Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:32:22.623 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:32:24.623 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1324, "optime" : { "t" : 1358379145000, "i" : 2771 }, "optimeDate" : ISODate("2013-01-16T23:32:25Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1313, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1313, "optime" : { "t" : 1358379141000, "i" : 2797 }, "optimeDate" : ISODate("2013-01-16T23:32:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:24Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1323, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1310, "optime" : { "t" : 1358379145000, "i" : 2137 }, "optimeDate" : ISODate("2013-01-16T23:32:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:25Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1310, "optime" : { "t" : 1358379141000, "i" : 2797 }, "optimeDate" : ISODate("2013-01-16T23:32:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1310, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1322, "optime" : { "t" : 1358379145000, "i" : 2137 }, "optimeDate" : ISODate("2013-01-16T23:32:25Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:32:26.049 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:32:26.624 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:32:28.624 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1328, "optime" : { "t" : 1358379149000, "i" : 4226 }, "optimeDate" : ISODate("2013-01-16T23:32:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1317, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1317, "optime" : { "t" : 1358379149000, "i" : 1098 }, "optimeDate" : ISODate("2013-01-16T23:32:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1327, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1314, "optime" : { "t" : 1358379149000, "i" : 2818 }, "optimeDate" : ISODate("2013-01-16T23:32:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1314, "optime" : { "t" : 1358379145000, "i" : 2771 }, "optimeDate" : ISODate("2013-01-16T23:32:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:29Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1314, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:29Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1326, "optime" : { "t" : 1358379149000, "i" : 4226 }, "optimeDate" : ISODate("2013-01-16T23:32:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:32:30.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:32:30.624 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:32:31.050 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:32:32.625 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:32:33.242 [conn82] end connection 127.0.0.3:56193 (3 connections now open) m31002| Wed Jan 16 18:32:33.242 [initandlisten] connection accepted from 127.0.0.3:59599 #83 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1332, "optime" : { "t" : 1358379153000, "i" : 4522 }, "optimeDate" : ISODate("2013-01-16T23:32:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1321, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1321, "optime" : { "t" : 1358379149000, "i" : 4226 }, "optimeDate" : ISODate("2013-01-16T23:32:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:32Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1331, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1318, "optime" : { "t" : 1358379153000, "i" : 2391 }, "optimeDate" : ISODate("2013-01-16T23:32:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1318, "optime" : { "t" : 1358379149000, "i" : 4226 }, "optimeDate" : ISODate("2013-01-16T23:32:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1318, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:33Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1330, "optime" : { "t" : 1358379153000, "i" : 4228 }, "optimeDate" : ISODate("2013-01-16T23:32:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:32:34.059 [conn81] end connection 127.0.0.2:39181 (3 connections now open) m31002| Wed Jan 16 18:32:34.559 [initandlisten] connection accepted from 127.0.0.2:38441 #84 (4 connections now open) m31000| Wed Jan 16 18:32:34.625 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:32:36.051 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:32:36.625 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1336, "optime" : { "t" : 1358379157000, "i" : 5184 }, "optimeDate" : ISODate("2013-01-16T23:32:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1325, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1325, "optime" : { "t" : 1358379157000, "i" : 374 }, "optimeDate" : ISODate("2013-01-16T23:32:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1335, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1322, "optime" : { "t" : 1358379157000, "i" : 1934 }, "optimeDate" : ISODate("2013-01-16T23:32:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1322, "optime" : { "t" : 1358379153000, "i" : 4522 }, "optimeDate" : ISODate("2013-01-16T23:32:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:37Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1322, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1334, "optime" : { "t" : 1358379157000, "i" : 5161 }, "optimeDate" : ISODate("2013-01-16T23:32:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:32:38.551 [conn80] end connection 127.0.0.4:42861 (2 connections now open) m31000| Wed Jan 16 18:32:38.626 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:32:39.051 [initandlisten] connection accepted from 127.0.0.4:45420 #82 (3 connections now open) m31000| Wed Jan 16 18:32:40.626 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:32:41.552 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1340, "optime" : { "t" : 1358379161000, "i" : 6563 }, "optimeDate" : ISODate("2013-01-16T23:32:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1329, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1329, "optime" : { "t" : 1358379157000, "i" : 5184 }, "optimeDate" : ISODate("2013-01-16T23:32:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:40Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1339, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1326, "optime" : { "t" : 1358379161000, "i" : 1854 }, "optimeDate" : ISODate("2013-01-16T23:32:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:41Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:41Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1326, "optime" : { "t" : 1358379157000, "i" : 5184 }, "optimeDate" : ISODate("2013-01-16T23:32:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1326, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1338, "optime" : { "t" : 1358379161000, "i" : 6563 }, "optimeDate" : ISODate("2013-01-16T23:32:41Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:32:42.626 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:32:44.627 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1344, "optime" : { "t" : 1358379165000, "i" : 8304 }, "optimeDate" : ISODate("2013-01-16T23:32:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1333, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1333, "optime" : { "t" : 1358379161000, "i" : 6563 }, "optimeDate" : ISODate("2013-01-16T23:32:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1343, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1330, "optime" : { "t" : 1358379165000, "i" : 2058 }, "optimeDate" : ISODate("2013-01-16T23:32:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1330, "optime" : { "t" : 1358379161000, "i" : 6563 }, "optimeDate" : ISODate("2013-01-16T23:32:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1330, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1342, "optime" : { "t" : 1358379165000, "i" : 7804 }, "optimeDate" : ISODate("2013-01-16T23:32:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:32:46.097 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:32:46.627 [conn81] end connection 127.0.0.2:45204 (2 connections now open) m31001| Wed Jan 16 18:32:46.627 [initandlisten] connection accepted from 127.0.0.2:60242 #83 (3 connections now open) m31000| Wed Jan 16 18:32:46.627 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:32:48.628 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1349, "optime" : { "t" : 1358379170000, "i" : 216 }, "optimeDate" : ISODate("2013-01-16T23:32:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1338, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1338, "optime" : { "t" : 1358379169000, "i" : 3828 }, "optimeDate" : ISODate("2013-01-16T23:32:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1348, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1335, "optime" : { "t" : 1358379169000, "i" : 1811 }, "optimeDate" : ISODate("2013-01-16T23:32:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1335, "optime" : { "t" : 1358379165000, "i" : 8304 }, "optimeDate" : ISODate("2013-01-16T23:32:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:49Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1335, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1347, "optime" : { "t" : 1358379170000, "i" : 216 }, "optimeDate" : ISODate("2013-01-16T23:32:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:32:50.098 [conn55] end connection 127.0.0.4:55634 (4 connections now open) m31000| Wed Jan 16 18:32:50.098 [initandlisten] connection accepted from 127.0.0.4:58937 #56 (5 connections now open) m31000| Wed Jan 16 18:32:50.628 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:32:51.553 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:32:52.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:32:52.629 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1353, "optime" : { "t" : 1358379174000, "i" : 1301 }, "optimeDate" : ISODate("2013-01-16T23:32:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1342, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1342, "optime" : { "t" : 1358379170000, "i" : 216 }, "optimeDate" : ISODate("2013-01-16T23:32:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:54Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1352, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1339, "optime" : { "t" : 1358379173000, "i" : 1833 }, "optimeDate" : ISODate("2013-01-16T23:32:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:54Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1339, "optime" : { "t" : 1358379174000, "i" : 315 }, "optimeDate" : ISODate("2013-01-16T23:32:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1339, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1351, "optime" : { "t" : 1358379174000, "i" : 992 }, "optimeDate" : ISODate("2013-01-16T23:32:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:32:54.629 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:32:56.099 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:32:56.629 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1357, "optime" : { "t" : 1358379178000, "i" : 2524 }, "optimeDate" : ISODate("2013-01-16T23:32:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1346, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1346, "optime" : { "t" : 1358379174000, "i" : 1301 }, "optimeDate" : ISODate("2013-01-16T23:32:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:58Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1356, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1343, "optime" : { "t" : 1358379177000, "i" : 548 }, "optimeDate" : ISODate("2013-01-16T23:32:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:32:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1343, "optime" : { "t" : 1358379178000, "i" : 1057 }, "optimeDate" : ISODate("2013-01-16T23:32:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1343, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:32:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1355, "optime" : { "t" : 1358379178000, "i" : 2524 }, "optimeDate" : ISODate("2013-01-16T23:32:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:32:58.630 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:33:00.630 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:33:01.554 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:33:02.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1361, "optime" : { "t" : 1358379182000, "i" : 3966 }, "optimeDate" : ISODate("2013-01-16T23:33:02Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1350, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1350, "optime" : { "t" : 1358379182000, "i" : 318 }, "optimeDate" : ISODate("2013-01-16T23:33:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:02Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1360, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1347, "optime" : { "t" : 1358379178000, "i" : 2524 }, "optimeDate" : ISODate("2013-01-16T23:32:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:01Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1347, "optime" : { "t" : 1358379182000, "i" : 820 }, "optimeDate" : ISODate("2013-01-16T23:33:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1347, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1359, "optime" : { "t" : 1358379182000, "i" : 3966 }, "optimeDate" : ISODate("2013-01-16T23:33:02Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:33:02.631 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:33:03.247 [conn83] end connection 127.0.0.3:59599 (3 connections now open) m31002| Wed Jan 16 18:33:03.247 [initandlisten] connection accepted from 127.0.0.3:40821 #85 (4 connections now open) m31000| Wed Jan 16 18:33:04.631 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:33:06.101 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:33:06.631 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1365, "optime" : { "t" : 1358379186000, "i" : 5596 }, "optimeDate" : ISODate("2013-01-16T23:33:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1354, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1354, "optime" : { "t" : 1358379182000, "i" : 3966 }, "optimeDate" : ISODate("2013-01-16T23:33:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:06Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1364, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1351, "optime" : { "t" : 1358379182000, "i" : 3966 }, "optimeDate" : ISODate("2013-01-16T23:33:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:06Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1351, "optime" : { "t" : 1358379186000, "i" : 402 }, "optimeDate" : ISODate("2013-01-16T23:33:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1351, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1363, "optime" : { "t" : 1358379186000, "i" : 5596 }, "optimeDate" : ISODate("2013-01-16T23:33:06Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:33:08.632 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:33:10.632 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1369, "optime" : { "t" : 1358379190000, "i" : 6944 }, "optimeDate" : ISODate("2013-01-16T23:33:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1358, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1358, "optime" : { "t" : 1358379186000, "i" : 5596 }, "optimeDate" : ISODate("2013-01-16T23:33:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:10Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:32:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1368, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1355, "optime" : { "t" : 1358379186000, "i" : 5596 }, "optimeDate" : ISODate("2013-01-16T23:33:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1355, "optime" : { "t" : 1358379190000, "i" : 923 }, "optimeDate" : ISODate("2013-01-16T23:33:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1355, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:09Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1367, "optime" : { "t" : 1358379190000, "i" : 6944 }, "optimeDate" : ISODate("2013-01-16T23:33:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:33:11.556 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:33:12.064 [conn84] end connection 127.0.0.2:38441 (3 connections now open) m31002| Wed Jan 16 18:33:12.564 [initandlisten] connection accepted from 127.0.0.2:41676 #86 (4 connections now open) m31000| Wed Jan 16 18:33:12.632 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:33:14.633 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1373, "optime" : { "t" : 1358379194000, "i" : 8440 }, "optimeDate" : ISODate("2013-01-16T23:33:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1362, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1362, "optime" : { "t" : 1358379190000, "i" : 6944 }, "optimeDate" : ISODate("2013-01-16T23:33:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:14Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1372, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1359, "optime" : { "t" : 1358379190000, "i" : 6944 }, "optimeDate" : ISODate("2013-01-16T23:33:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:14Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1359, "optime" : { "t" : 1358379194000, "i" : 519 }, "optimeDate" : ISODate("2013-01-16T23:33:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1359, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1371, "optime" : { "t" : 1358379194000, "i" : 8440 }, "optimeDate" : ISODate("2013-01-16T23:33:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:33:16.102 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:33:16.556 [conn82] end connection 127.0.0.4:45420 (2 connections now open) m31001| Wed Jan 16 18:33:16.633 [conn83] end connection 127.0.0.2:60242 (1 connection now open) m31001| Wed Jan 16 18:33:16.633 [initandlisten] connection accepted from 127.0.0.2:60760 #84 (2 connections now open) m31000| Wed Jan 16 18:33:16.633 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:33:17.056 [initandlisten] connection accepted from 127.0.0.4:48956 #85 (3 connections now open) m31000| Wed Jan 16 18:33:18.634 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1378, "optime" : { "t" : 1358379199000, "i" : 839 }, "optimeDate" : ISODate("2013-01-16T23:33:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1367, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1367, "optime" : { "t" : 1358379194000, "i" : 8440 }, "optimeDate" : ISODate("2013-01-16T23:33:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:18Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1377, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1364, "optime" : { "t" : 1358379194000, "i" : 8440 }, "optimeDate" : ISODate("2013-01-16T23:33:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1364, "optime" : { "t" : 1358379198000, "i" : 993 }, "optimeDate" : ISODate("2013-01-16T23:33:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1364, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1376, "optime" : { "t" : 1358379199000, "i" : 209 }, "optimeDate" : ISODate("2013-01-16T23:33:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:33:20.103 [conn56] end connection 127.0.0.4:58937 (4 connections now open) m31000| Wed Jan 16 18:33:20.103 [initandlisten] connection accepted from 127.0.0.4:36531 #57 (5 connections now open) m31000| Wed Jan 16 18:33:20.634 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:33:22.057 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:33:22.634 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1382, "optime" : { "t" : 1358379203000, "i" : 2553 }, "optimeDate" : ISODate("2013-01-16T23:33:23Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1371, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1371, "optime" : { "t" : 1358379202000, "i" : 3997 }, "optimeDate" : ISODate("2013-01-16T23:33:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1381, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1368, "optime" : { "t" : 1358379199000, "i" : 839 }, "optimeDate" : ISODate("2013-01-16T23:33:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:22Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1368, "optime" : { "t" : 1358379199000, "i" : 839 }, "optimeDate" : ISODate("2013-01-16T23:33:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1368, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1380, "optime" : { "t" : 1358379203000, "i" : 2524 }, "optimeDate" : ISODate("2013-01-16T23:33:23Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:33:24.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:33:24.635 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:33:26.635 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:33:27.058 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1386, "optime" : { "t" : 1358379207000, "i" : 3550 }, "optimeDate" : ISODate("2013-01-16T23:33:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1375, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1375, "optime" : { "t" : 1358379203000, "i" : 2553 }, "optimeDate" : ISODate("2013-01-16T23:33:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:26Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1385, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1372, "optime" : { "t" : 1358379207000, "i" : 2650 }, "optimeDate" : ISODate("2013-01-16T23:33:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1372, "optime" : { "t" : 1358379203000, "i" : 2553 }, "optimeDate" : ISODate("2013-01-16T23:33:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1372, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:27Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1384, "optime" : { "t" : 1358379207000, "i" : 3550 }, "optimeDate" : ISODate("2013-01-16T23:33:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:33:28.635 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:33:30.636 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1390, "optime" : { "t" : 1358379211000, "i" : 4166 }, "optimeDate" : ISODate("2013-01-16T23:33:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1379, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1379, "optime" : { "t" : 1358379207000, "i" : 3550 }, "optimeDate" : ISODate("2013-01-16T23:33:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:30Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1389, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1376, "optime" : { "t" : 1358379211000, "i" : 1370 }, "optimeDate" : ISODate("2013-01-16T23:33:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1376, "optime" : { "t" : 1358379207000, "i" : 3550 }, "optimeDate" : ISODate("2013-01-16T23:33:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1376, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:31Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1388, "optime" : { "t" : 1358379211000, "i" : 4077 }, "optimeDate" : ISODate("2013-01-16T23:33:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:33:32.059 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:33:32.636 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:33:33.253 [conn85] end connection 127.0.0.3:40821 (3 connections now open) m31002| Wed Jan 16 18:33:33.253 [initandlisten] connection accepted from 127.0.0.3:54824 #87 (4 connections now open) m31001| Wed Jan 16 18:33:34.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:33:34.636 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1394, "optime" : { "t" : 1358379215000, "i" : 5812 }, "optimeDate" : ISODate("2013-01-16T23:33:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1383, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1383, "optime" : { "t" : 1358379215000, "i" : 987 }, "optimeDate" : ISODate("2013-01-16T23:33:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1393, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1380, "optime" : { "t" : 1358379215000, "i" : 2666 }, "optimeDate" : ISODate("2013-01-16T23:33:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1380, "optime" : { "t" : 1358379211000, "i" : 4166 }, "optimeDate" : ISODate("2013-01-16T23:33:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:35Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1380, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:35Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1392, "optime" : { "t" : 1358379215000, "i" : 5486 }, "optimeDate" : ISODate("2013-01-16T23:33:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:33:36.637 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:33:37.059 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:33:38.637 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:33:39.658 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:410385 205ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1398, "optime" : { "t" : 1358379219000, "i" : 6949 }, "optimeDate" : ISODate("2013-01-16T23:33:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1387, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1387, "optime" : { "t" : 1358379215000, "i" : 5812 }, "optimeDate" : ISODate("2013-01-16T23:33:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:38Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1397, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1384, "optime" : { "t" : 1358379219000, "i" : 2364 }, "optimeDate" : ISODate("2013-01-16T23:33:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:39Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1384, "optime" : { "t" : 1358379215000, "i" : 5812 }, "optimeDate" : ISODate("2013-01-16T23:33:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1384, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1396, "optime" : { "t" : 1358379219000, "i" : 6306 }, "optimeDate" : ISODate("2013-01-16T23:33:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:33:40.637 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:33:42.060 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:33:42.638 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1403, "optime" : { "t" : 1358379224000, "i" : 42 }, "optimeDate" : ISODate("2013-01-16T23:33:44Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1392, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1392, "optime" : { "t" : 1358379219000, "i" : 6949 }, "optimeDate" : ISODate("2013-01-16T23:33:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1402, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1389, "optime" : { "t" : 1358379223000, "i" : 2230 }, "optimeDate" : ISODate("2013-01-16T23:33:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1389, "optime" : { "t" : 1358379219000, "i" : 6949 }, "optimeDate" : ISODate("2013-01-16T23:33:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1389, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:43Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1401, "optime" : { "t" : 1358379224000, "i" : 42 }, "optimeDate" : ISODate("2013-01-16T23:33:44Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:33:44.638 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:33:46.638 [conn84] end connection 127.0.0.2:60760 (2 connections now open) m31001| Wed Jan 16 18:33:46.639 [initandlisten] connection accepted from 127.0.0.2:47583 #86 (3 connections now open) m31000| Wed Jan 16 18:33:46.639 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:33:47.061 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1407, "optime" : { "t" : 1358379228000, "i" : 1015 }, "optimeDate" : ISODate("2013-01-16T23:33:48Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1396, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1396, "optime" : { "t" : 1358379227000, "i" : 4992 }, "optimeDate" : ISODate("2013-01-16T23:33:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:48Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1406, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1393, "optime" : { "t" : 1358379227000, "i" : 1963 }, "optimeDate" : ISODate("2013-01-16T23:33:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1393, "optime" : { "t" : 1358379228000, "i" : 351 }, "optimeDate" : ISODate("2013-01-16T23:33:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1393, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1405, "optime" : { "t" : 1358379228000, "i" : 822 }, "optimeDate" : ISODate("2013-01-16T23:33:48Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:33:48.639 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:33:50.069 [conn86] end connection 127.0.0.2:41676 (3 connections now open) m31000| Wed Jan 16 18:33:50.109 [conn57] end connection 127.0.0.4:36531 (4 connections now open) m31000| Wed Jan 16 18:33:50.109 [initandlisten] connection accepted from 127.0.0.4:56241 #58 (5 connections now open) m31002| Wed Jan 16 18:33:50.570 [initandlisten] connection accepted from 127.0.0.2:40535 #88 (4 connections now open) m31000| Wed Jan 16 18:33:50.640 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:33:52.061 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1411, "optime" : { "t" : 1358379232000, "i" : 2778 }, "optimeDate" : ISODate("2013-01-16T23:33:52Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1400, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1400, "optime" : { "t" : 1358379228000, "i" : 1015 }, "optimeDate" : ISODate("2013-01-16T23:33:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:52Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1410, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1397, "optime" : { "t" : 1358379231000, "i" : 1139 }, "optimeDate" : ISODate("2013-01-16T23:33:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:52Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1397, "optime" : { "t" : 1358379232000, "i" : 1236 }, "optimeDate" : ISODate("2013-01-16T23:33:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1397, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1409, "optime" : { "t" : 1358379232000, "i" : 2728 }, "optimeDate" : ISODate("2013-01-16T23:33:52Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:33:52.640 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:33:54.561 [conn85] end connection 127.0.0.4:48956 (2 connections now open) m31000| Wed Jan 16 18:33:54.641 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:33:55.061 [initandlisten] connection accepted from 127.0.0.4:35901 #87 (3 connections now open) m31001| Wed Jan 16 18:33:56.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1415, "optime" : { "t" : 1358379236000, "i" : 4108 }, "optimeDate" : ISODate("2013-01-16T23:33:56Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1404, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1404, "optime" : { "t" : 1358379235000, "i" : 1840 }, "optimeDate" : ISODate("2013-01-16T23:33:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:56Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1414, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1401, "optime" : { "t" : 1358379232000, "i" : 2778 }, "optimeDate" : ISODate("2013-01-16T23:33:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:33:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1401, "optime" : { "t" : 1358379236000, "i" : 1053 }, "optimeDate" : ISODate("2013-01-16T23:33:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1401, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:33:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1413, "optime" : { "t" : 1358379236000, "i" : 4108 }, "optimeDate" : ISODate("2013-01-16T23:33:56Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:33:56.641 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:33:57.562 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:33:58.641 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:34:00.642 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1419, "optime" : { "t" : 1358379240000, "i" : 5665 }, "optimeDate" : ISODate("2013-01-16T23:34:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1408, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1408, "optime" : { "t" : 1358379240000, "i" : 4795 }, "optimeDate" : ISODate("2013-01-16T23:34:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1418, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1405, "optime" : { "t" : 1358379236000, "i" : 4108 }, "optimeDate" : ISODate("2013-01-16T23:33:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1405, "optime" : { "t" : 1358379240000, "i" : 1009 }, "optimeDate" : ISODate("2013-01-16T23:34:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1405, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1417, "optime" : { "t" : 1358379240000, "i" : 5528 }, "optimeDate" : ISODate("2013-01-16T23:34:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:34:02.111 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:34:02.642 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:34:03.258 [conn87] end connection 127.0.0.3:54824 (3 connections now open) m31002| Wed Jan 16 18:34:03.258 [initandlisten] connection accepted from 127.0.0.3:53218 #89 (4 connections now open) m31000| Wed Jan 16 18:34:04.642 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1423, "optime" : { "t" : 1358379244000, "i" : 7361 }, "optimeDate" : ISODate("2013-01-16T23:34:04Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1412, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1412, "optime" : { "t" : 1358379240000, "i" : 5665 }, "optimeDate" : ISODate("2013-01-16T23:34:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:04Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1422, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1409, "optime" : { "t" : 1358379240000, "i" : 5665 }, "optimeDate" : ISODate("2013-01-16T23:34:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1409, "optime" : { "t" : 1358379244000, "i" : 1281 }, "optimeDate" : ISODate("2013-01-16T23:34:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1409, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:03Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1421, "optime" : { "t" : 1358379244000, "i" : 7361 }, "optimeDate" : ISODate("2013-01-16T23:34:04Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:34:06.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:34:06.643 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:34:07.564 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:34:08.643 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1427, "optime" : { "t" : 1358379248000, "i" : 8343 }, "optimeDate" : ISODate("2013-01-16T23:34:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1416, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1416, "optime" : { "t" : 1358379248000, "i" : 318 }, "optimeDate" : ISODate("2013-01-16T23:34:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:08Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1426, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1413, "optime" : { "t" : 1358379244000, "i" : 7361 }, "optimeDate" : ISODate("2013-01-16T23:34:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:07Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1413, "optime" : { "t" : 1358379248000, "i" : 733 }, "optimeDate" : ISODate("2013-01-16T23:34:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1413, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1425, "optime" : { "t" : 1358379248000, "i" : 7754 }, "optimeDate" : ISODate("2013-01-16T23:34:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:34:10.644 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:34:12.113 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:34:12.644 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1432, "optime" : { "t" : 1358379253000, "i" : 1308 }, "optimeDate" : ISODate("2013-01-16T23:34:13Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1421, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1421, "optime" : { "t" : 1358379253000, "i" : 822 }, "optimeDate" : ISODate("2013-01-16T23:34:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:33:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1431, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1418, "optime" : { "t" : 1358379248000, "i" : 8343 }, "optimeDate" : ISODate("2013-01-16T23:34:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:12Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1418, "optime" : { "t" : 1358379252000, "i" : 1259 }, "optimeDate" : ISODate("2013-01-16T23:34:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1418, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1430, "optime" : { "t" : 1358379253000, "i" : 1308 }, "optimeDate" : ISODate("2013-01-16T23:34:13Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:34:14.644 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:34:16.644 [conn86] end connection 127.0.0.2:47583 (2 connections now open) m31001| Wed Jan 16 18:34:16.645 [initandlisten] connection accepted from 127.0.0.2:56976 #88 (3 connections now open) m31000| Wed Jan 16 18:34:16.645 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1436, "optime" : { "t" : 1358379257000, "i" : 2841 }, "optimeDate" : ISODate("2013-01-16T23:34:17Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1425, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1425, "optime" : { "t" : 1358379253000, "i" : 1308 }, "optimeDate" : ISODate("2013-01-16T23:34:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1435, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1422, "optime" : { "t" : 1358379257000, "i" : 1705 }, "optimeDate" : ISODate("2013-01-16T23:34:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1422, "optime" : { "t" : 1358379253000, "i" : 1308 }, "optimeDate" : ISODate("2013-01-16T23:34:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1422, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1434, "optime" : { "t" : 1358379257000, "i" : 2841 }, "optimeDate" : ISODate("2013-01-16T23:34:17Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:34:17.565 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:34:18.645 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:34:20.114 [conn58] end connection 127.0.0.4:56241 (4 connections now open) m31000| Wed Jan 16 18:34:20.114 [initandlisten] connection accepted from 127.0.0.4:54724 #59 (5 connections now open) m31000| Wed Jan 16 18:34:20.645 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1440, "optime" : { "t" : 1358379261000, "i" : 3885 }, "optimeDate" : ISODate("2013-01-16T23:34:21Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1429, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1429, "optime" : { "t" : 1358379260000, "i" : 2077 }, "optimeDate" : ISODate("2013-01-16T23:34:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1439, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1426, "optime" : { "t" : 1358379261000, "i" : 2086 }, "optimeDate" : ISODate("2013-01-16T23:34:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1426, "optime" : { "t" : 1358379257000, "i" : 2841 }, "optimeDate" : ISODate("2013-01-16T23:34:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1426, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:21Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1438, "optime" : { "t" : 1358379261000, "i" : 3551 }, "optimeDate" : ISODate("2013-01-16T23:34:21Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:34:22.115 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:34:22.646 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:34:24.646 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1444, "optime" : { "t" : 1358379265000, "i" : 5494 }, "optimeDate" : ISODate("2013-01-16T23:34:25Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1433, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1433, "optime" : { "t" : 1358379261000, "i" : 3885 }, "optimeDate" : ISODate("2013-01-16T23:34:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:24Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1443, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1430, "optime" : { "t" : 1358379265000, "i" : 2858 }, "optimeDate" : ISODate("2013-01-16T23:34:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:25Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1430, "optime" : { "t" : 1358379261000, "i" : 3885 }, "optimeDate" : ISODate("2013-01-16T23:34:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1430, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1442, "optime" : { "t" : 1358379265000, "i" : 5494 }, "optimeDate" : ISODate("2013-01-16T23:34:25Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:34:26.647 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:34:27.566 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:34:28.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:34:28.075 [conn88] end connection 127.0.0.2:40535 (3 connections now open) m31002| Wed Jan 16 18:34:28.575 [initandlisten] connection accepted from 127.0.0.2:58084 #90 (4 connections now open) m31000| Wed Jan 16 18:34:28.647 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1448, "optime" : { "t" : 1358379269000, "i" : 6344 }, "optimeDate" : ISODate("2013-01-16T23:34:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1437, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1437, "optime" : { "t" : 1358379268000, "i" : 5 }, "optimeDate" : ISODate("2013-01-16T23:34:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1447, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1434, "optime" : { "t" : 1358379269000, "i" : 1600 }, "optimeDate" : ISODate("2013-01-16T23:34:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1434, "optime" : { "t" : 1358379265000, "i" : 5494 }, "optimeDate" : ISODate("2013-01-16T23:34:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1434, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:29Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1446, "optime" : { "t" : 1358379269000, "i" : 6344 }, "optimeDate" : ISODate("2013-01-16T23:34:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:34:30.647 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:34:32.117 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:34:32.567 [conn87] end connection 127.0.0.4:35901 (2 connections now open) m31000| Wed Jan 16 18:34:32.648 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:34:33.067 [initandlisten] connection accepted from 127.0.0.4:46636 #89 (3 connections now open) m31002| Wed Jan 16 18:34:33.264 [conn89] end connection 127.0.0.3:53218 (3 connections now open) m31002| Wed Jan 16 18:34:33.264 [initandlisten] connection accepted from 127.0.0.3:40964 #91 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1452, "optime" : { "t" : 1358379273000, "i" : 7349 }, "optimeDate" : ISODate("2013-01-16T23:34:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1441, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1441, "optime" : { "t" : 1358379273000, "i" : 4295 }, "optimeDate" : ISODate("2013-01-16T23:34:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1451, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1438, "optime" : { "t" : 1358379273000, "i" : 1734 }, "optimeDate" : ISODate("2013-01-16T23:34:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1438, "optime" : { "t" : 1358379269000, "i" : 6344 }, "optimeDate" : ISODate("2013-01-16T23:34:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:33Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1438, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:33Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1450, "optime" : { "t" : 1358379273000, "i" : 6804 }, "optimeDate" : ISODate("2013-01-16T23:34:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:34:34.648 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:34:36.648 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1456, "optime" : { "t" : 1358379277000, "i" : 8524 }, "optimeDate" : ISODate("2013-01-16T23:34:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1445, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1445, "optime" : { "t" : 1358379273000, "i" : 7349 }, "optimeDate" : ISODate("2013-01-16T23:34:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:36Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1455, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1442, "optime" : { "t" : 1358379277000, "i" : 2138 }, "optimeDate" : ISODate("2013-01-16T23:34:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1442, "optime" : { "t" : 1358379273000, "i" : 7349 }, "optimeDate" : ISODate("2013-01-16T23:34:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1442, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1454, "optime" : { "t" : 1358379277000, "i" : 8524 }, "optimeDate" : ISODate("2013-01-16T23:34:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:34:38.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:34:38.068 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:34:38.649 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:34:40.649 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1461, "optime" : { "t" : 1358379282000, "i" : 878 }, "optimeDate" : ISODate("2013-01-16T23:34:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1450, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1450, "optime" : { "t" : 1358379281000, "i" : 576 }, "optimeDate" : ISODate("2013-01-16T23:34:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:42Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1460, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1447, "optime" : { "t" : 1358379281000, "i" : 1896 }, "optimeDate" : ISODate("2013-01-16T23:34:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1447, "optime" : { "t" : 1358379282000, "i" : 708 }, "optimeDate" : ISODate("2013-01-16T23:34:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1447, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:41Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1459, "optime" : { "t" : 1358379282000, "i" : 878 }, "optimeDate" : ISODate("2013-01-16T23:34:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:34:42.650 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:34:43.069 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:34:44.650 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1465, "optime" : { "t" : 1358379286000, "i" : 2610 }, "optimeDate" : ISODate("2013-01-16T23:34:46Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1454, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1454, "optime" : { "t" : 1358379286000, "i" : 843 }, "optimeDate" : ISODate("2013-01-16T23:34:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:46Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1464, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1451, "optime" : { "t" : 1358379285000, "i" : 285 }, "optimeDate" : ISODate("2013-01-16T23:34:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:45Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1451, "optime" : { "t" : 1358379286000, "i" : 951 }, "optimeDate" : ISODate("2013-01-16T23:34:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1451, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1463, "optime" : { "t" : 1358379286000, "i" : 2161 }, "optimeDate" : ISODate("2013-01-16T23:34:46Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:34:46.650 [conn88] end connection 127.0.0.2:56976 (2 connections now open) m31001| Wed Jan 16 18:34:46.650 [initandlisten] connection accepted from 127.0.0.2:43355 #90 (3 connections now open) m31000| Wed Jan 16 18:34:46.651 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:34:48.069 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:34:48.651 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:34:49.712 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:494787 247ms m31000| Wed Jan 16 18:34:50.120 [conn59] end connection 127.0.0.4:54724 (4 connections now open) m31000| Wed Jan 16 18:34:50.120 [initandlisten] connection accepted from 127.0.0.4:47199 #60 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1469, "optime" : { "t" : 1358379290000, "i" : 3090 }, "optimeDate" : ISODate("2013-01-16T23:34:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1458, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1458, "optime" : { "t" : 1358379286000, "i" : 2610 }, "optimeDate" : ISODate("2013-01-16T23:34:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:50Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1468, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1455, "optime" : { "t" : 1358379286000, "i" : 2610 }, "optimeDate" : ISODate("2013-01-16T23:34:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1455, "optime" : { "t" : 1358379290000, "i" : 1184 }, "optimeDate" : ISODate("2013-01-16T23:34:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1455, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1467, "optime" : { "t" : 1358379290000, "i" : 3090 }, "optimeDate" : ISODate("2013-01-16T23:34:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:34:50.652 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:34:52.652 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:34:53.070 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1473, "optime" : { "t" : 1358379294000, "i" : 4265 }, "optimeDate" : ISODate("2013-01-16T23:34:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1462, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1462, "optime" : { "t" : 1358379293000, "i" : 1726 }, "optimeDate" : ISODate("2013-01-16T23:34:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:54Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1472, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1459, "optime" : { "t" : 1358379290000, "i" : 3090 }, "optimeDate" : ISODate("2013-01-16T23:34:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1459, "optime" : { "t" : 1358379294000, "i" : 1104 }, "optimeDate" : ISODate("2013-01-16T23:34:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1459, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:53Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1471, "optime" : { "t" : 1358379294000, "i" : 4265 }, "optimeDate" : ISODate("2013-01-16T23:34:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:34:54.652 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:34:56.653 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:34:58.071 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1477, "optime" : { "t" : 1358379298000, "i" : 5430 }, "optimeDate" : ISODate("2013-01-16T23:34:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1466, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1466, "optime" : { "t" : 1358379298000, "i" : 4879 }, "optimeDate" : ISODate("2013-01-16T23:34:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1476, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1463, "optime" : { "t" : 1358379294000, "i" : 4265 }, "optimeDate" : ISODate("2013-01-16T23:34:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:58Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 m31000| Wed Jan 16 18:34:58.653 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:34:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1463, "optime" : { "t" : 1358379298000, "i" : 1029 }, "optimeDate" : ISODate("2013-01-16T23:34:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:34:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1463, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1475, "optime" : { "t" : 1358379298000, "i" : 5430 }, "optimeDate" : ISODate("2013-01-16T23:34:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:35:00.007 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:35:00.654 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:35:31.742 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:35:32.274 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1511, "optime" : { "t" : 1358379332000, "i" : 3810 }, "optimeDate" : ISODate("2013-01-16T23:35:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1500, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1500, "optime" : { "t" : 1358379298000, "i" : 5430 }, "optimeDate" : ISODate("2013-01-16T23:34:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:31Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1510, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1497, "optime" : { "t" : 1358379298000, "i" : 5430 }, "optimeDate" : ISODate("2013-01-16T23:34:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1497, "optime" : { "t" : 1358379331000, "i" : 4330 }, "optimeDate" : ISODate("2013-01-16T23:35:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1497, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:30Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1509, "optime" : { "t" : 1358379332000, "i" : 3810 }, "optimeDate" : ISODate("2013-01-16T23:35:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:35:32.890 [conn91] end connection 127.0.0.3:40964 (3 connections now open) m31002| Wed Jan 16 18:35:32.890 [initandlisten] connection accepted from 127.0.0.3:59527 #92 (4 connections now open) m31000| Wed Jan 16 18:35:34.275 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:35:35.700 [conn90] end connection 127.0.0.2:58084 (3 connections now open) m31002| Wed Jan 16 18:35:36.200 [initandlisten] connection accepted from 127.0.0.2:38742 #93 (4 connections now open) m31000| Wed Jan 16 18:35:36.275 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1515, "optime" : { "t" : 1358379336000, "i" : 4940 }, "optimeDate" : ISODate("2013-01-16T23:35:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1504, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1504, "optime" : { "t" : 1358379336000, "i" : 1408 }, "optimeDate" : ISODate("2013-01-16T23:35:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1514, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1501, "optime" : { "t" : 1358379332000, "i" : 3810 }, "optimeDate" : ISODate("2013-01-16T23:35:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:35Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1501, "optime" : { "t" : 1358379335000, "i" : 2419 }, "optimeDate" : ISODate("2013-01-16T23:35:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1501, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1513, "optime" : { "t" : 1358379336000, "i" : 3989 }, "optimeDate" : ISODate("2013-01-16T23:35:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:35:37.693 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:35:38.275 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:35:39.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 18:35:40.193 [conn89] end connection 127.0.0.4:46636 (2 connections now open) m31000| Wed Jan 16 18:35:40.276 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:35:40.693 [initandlisten] connection accepted from 127.0.0.4:53173 #91 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1519, "optime" : { "t" : 1358379340000, "i" : 6141 }, "optimeDate" : ISODate("2013-01-16T23:35:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1508, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1508, "optime" : { "t" : 1358379336000, "i" : 4940 }, "optimeDate" : ISODate("2013-01-16T23:35:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:39Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1518, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1505, "optime" : { "t" : 1358379336000, "i" : 4940 }, "optimeDate" : ISODate("2013-01-16T23:35:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:40Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1505, "optime" : { "t" : 1358379339000, "i" : 1648 }, "optimeDate" : ISODate("2013-01-16T23:35:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1505, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1517, "optime" : { "t" : 1358379340000, "i" : 6073 }, "optimeDate" : ISODate("2013-01-16T23:35:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:35:42.276 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:35:43.193 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:35:44.276 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1523, "optime" : { "t" : 1358379344000, "i" : 8064 }, "optimeDate" : ISODate("2013-01-16T23:35:44Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1512, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1512, "optime" : { "t" : 1358379340000, "i" : 6141 }, "optimeDate" : ISODate("2013-01-16T23:35:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:43Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1522, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1509, "optime" : { "t" : 1358379340000, "i" : 6141 }, "optimeDate" : ISODate("2013-01-16T23:35:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:43Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1509, "optime" : { "t" : 1358379343000, "i" : 115 }, "optimeDate" : ISODate("2013-01-16T23:35:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1509, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1521, "optime" : { "t" : 1358379344000, "i" : 8064 }, "optimeDate" : ISODate("2013-01-16T23:35:44Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:35:46.277 [conn90] end connection 127.0.0.2:43355 (2 connections now open) m31001| Wed Jan 16 18:35:46.277 [initandlisten] connection accepted from 127.0.0.2:55429 #92 (3 connections now open) m31000| Wed Jan 16 18:35:46.277 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:35:48.194 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:35:48.277 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1527, "optime" : { "t" : 1358379348000, "i" : 8802 }, "optimeDate" : ISODate("2013-01-16T23:35:48Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1516, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1516, "optime" : { "t" : 1358379348000, "i" : 5766 }, "optimeDate" : ISODate("2013-01-16T23:35:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:34:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1526, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1513, "optime" : { "t" : 1358379348000, "i" : 7894 }, "optimeDate" : ISODate("2013-01-16T23:35:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1513, "optime" : { "t" : 1358379344000, "i" : 8064 }, "optimeDate" : ISODate("2013-01-16T23:35:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1513, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:48Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1525, "optime" : { "t" : 1358379348000, "i" : 7927 }, "optimeDate" : ISODate("2013-01-16T23:35:48Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:35:49.745 [conn60] end connection 127.0.0.4:47199 (4 connections now open) m31000| Wed Jan 16 18:35:49.745 [initandlisten] connection accepted from 127.0.0.4:41227 #61 (5 connections now open) m31000| Wed Jan 16 18:35:50.278 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:35:52.278 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:35:53.190 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:396984 198ms m31002| Wed Jan 16 18:35:53.195 [rsMgr] replSet info two primaries (transiently) { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1532, "optime" : { "t" : 1358379353000, "i" : 1556 }, "optimeDate" : ISODate("2013-01-16T23:35:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1521, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1521, "optime" : { "t" : 1358379348000, "i" : 8802 }, "optimeDate" : ISODate("2013-01-16T23:35:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:51Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1531, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1518, "optime" : { "t" : 1358379352000, "i" : 7053 }, "optimeDate" : ISODate("2013-01-16T23:35:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:53Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1518, "optime" : { "t" : 1358379348000, "i" : 8802 }, "optimeDate" : ISODate("2013-01-16T23:35:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1518, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1530, "optime" : { "t" : 1358379353000, "i" : 1556 }, "optimeDate" : ISODate("2013-01-16T23:35:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:35:54.278 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:35:56.279 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1536, "optime" : { "t" : 1358379357000, "i" : 3842 }, "optimeDate" : ISODate("2013-01-16T23:35:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1525, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1525, "optime" : { "t" : 1358379353000, "i" : 1556 }, "optimeDate" : ISODate("2013-01-16T23:35:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1535, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1522, "optime" : { "t" : 1358379356000, "i" : 5583 }, "optimeDate" : ISODate("2013-01-16T23:35:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:35:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1522, "optime" : { "t" : 1358379353000, "i" : 1556 }, "optimeDate" : ISODate("2013-01-16T23:35:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1522, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:35:56Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1534, "optime" : { "t" : 1358379357000, "i" : 3079 }, "optimeDate" : ISODate("2013-01-16T23:35:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:35:58.196 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:35:58.279 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:36:00.279 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1540, "optime" : { "t" : 1358379361000, "i" : 4295 }, "optimeDate" : ISODate("2013-01-16T23:36:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1529, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1529, "optime" : { "t" : 1358379361000, "i" : 1466 }, "optimeDate" : ISODate("2013-01-16T23:36:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1539, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1526, "optime" : { "t" : 1358379360000, "i" : 4561 }, "optimeDate" : ISODate("2013-01-16T23:36:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1526, "optime" : { "t" : 1358379357000, "i" : 3842 }, "optimeDate" : ISODate("2013-01-16T23:35:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:01Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1526, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1538, "optime" : { "t" : 1358379361000, "i" : 4213 }, "optimeDate" : ISODate("2013-01-16T23:36:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:36:01.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:36:02.280 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:36:02.895 [conn92] end connection 127.0.0.3:59527 (3 connections now open) m31002| Wed Jan 16 18:36:02.895 [initandlisten] connection accepted from 127.0.0.3:46872 #94 (4 connections now open) m31002| Wed Jan 16 18:36:03.196 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:36:04.280 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1544, "optime" : { "t" : 1358379365000, "i" : 6365 }, "optimeDate" : ISODate("2013-01-16T23:36:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1533, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1533, "optime" : { "t" : 1358379361000, "i" : 4295 }, "optimeDate" : ISODate("2013-01-16T23:36:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:05Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1543, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1530, "optime" : { "t" : 1358379364000, "i" : 2841 }, "optimeDate" : ISODate("2013-01-16T23:36:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:05Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1530, "optime" : { "t" : 1358379365000, "i" : 6248 }, "optimeDate" : ISODate("2013-01-16T23:36:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1530, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1542, "optime" : { "t" : 1358379365000, "i" : 6365 }, "optimeDate" : ISODate("2013-01-16T23:36:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:36:06.281 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:36:08.197 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:36:08.281 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1548, "optime" : { "t" : 1358379369000, "i" : 8446 }, "optimeDate" : ISODate("2013-01-16T23:36:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1537, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1537, "optime" : { "t" : 1358379365000, "i" : 6365 }, "optimeDate" : ISODate("2013-01-16T23:36:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:09Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1547, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1534, "optime" : { "t" : 1358379368000, "i" : 587 }, "optimeDate" : ISODate("2013-01-16T23:36:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1534, "optime" : { "t" : 1358379369000, "i" : 6536 }, "optimeDate" : ISODate("2013-01-16T23:36:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1534, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:08Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1546, "optime" : { "t" : 1358379369000, "i" : 7905 }, "optimeDate" : ISODate("2013-01-16T23:36:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:36:10.281 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:36:11.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:36:12.282 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:36:13.197 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:36:13.705 [conn93] end connection 127.0.0.2:38742 (3 connections now open) STATUS FROM : 127.0.0.2:31000 m31002| Wed Jan 16 18:36:14.205 [initandlisten] connection accepted from 127.0.0.2:45611 #95 (4 connections now open) m31000| Wed Jan 16 18:36:14.226 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:303500 151ms { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1553, "optime" : { "t" : 1358379374000, "i" : 890 }, "optimeDate" : ISODate("2013-01-16T23:36:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1542, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1542, "optime" : { "t" : 1358379374000, "i" : 707 }, "optimeDate" : ISODate("2013-01-16T23:36:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1552, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1539, "optime" : { "t" : 1358379369000, "i" : 8446 }, "optimeDate" : ISODate("2013-01-16T23:36:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:13Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1539, "optime" : { "t" : 1358379373000, "i" : 6879 }, "optimeDate" : ISODate("2013-01-16T23:36:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1539, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1551, "optime" : { "t" : 1358379374000, "i" : 707 }, "optimeDate" : ISODate("2013-01-16T23:36:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:36:14.282 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:36:16.283 [conn92] end connection 127.0.0.2:55429 (2 connections now open) m31001| Wed Jan 16 18:36:16.283 [initandlisten] connection accepted from 127.0.0.2:57391 #93 (3 connections now open) m31000| Wed Jan 16 18:36:16.283 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:36:18.198 [conn91] end connection 127.0.0.4:53173 (2 connections now open) m31000| Wed Jan 16 18:36:18.283 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1557, "optime" : { "t" : 1358379378000, "i" : 3455 }, "optimeDate" : ISODate("2013-01-16T23:36:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1546, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1546, "optime" : { "t" : 1358379374000, "i" : 890 }, "optimeDate" : ISODate("2013-01-16T23:36:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:17Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:35:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1556, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1543, "optime" : { "t" : 1358379374000, "i" : 890 }, "optimeDate" : ISODate("2013-01-16T23:36:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1543, "optime" : { "t" : 1358379377000, "i" : 4419 }, "optimeDate" : ISODate("2013-01-16T23:36:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1543, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:16Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1555, "optime" : { "t" : 1358379378000, "i" : 3455 }, "optimeDate" : ISODate("2013-01-16T23:36:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:36:18.698 [initandlisten] connection accepted from 127.0.0.4:59518 #94 (3 connections now open) m31002| Wed Jan 16 18:36:18.698 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:36:19.751 [conn61] end connection 127.0.0.4:41227 (4 connections now open) m31000| Wed Jan 16 18:36:19.751 [initandlisten] connection accepted from 127.0.0.4:60654 #62 (5 connections now open) m31000| Wed Jan 16 18:36:20.284 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:36:22.284 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1561, "optime" : { "t" : 1358379382000, "i" : 4656 }, "optimeDate" : ISODate("2013-01-16T23:36:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1550, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1550, "optime" : { "t" : 1358379381000, "i" : 2591 }, "optimeDate" : ISODate("2013-01-16T23:36:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:21Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1560, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1547, "optime" : { "t" : 1358379378000, "i" : 3455 }, "optimeDate" : ISODate("2013-01-16T23:36:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:21Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1547, "optime" : { "t" : 1358379381000, "i" : 3038 }, "optimeDate" : ISODate("2013-01-16T23:36:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1547, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1559, "optime" : { "t" : 1358379382000, "i" : 4656 }, "optimeDate" : ISODate("2013-01-16T23:36:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:36:23.699 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:36:24.284 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:36:26.285 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1565, "optime" : { "t" : 1358379386000, "i" : 6313 }, "optimeDate" : ISODate("2013-01-16T23:36:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1554, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1554, "optime" : { "t" : 1358379386000, "i" : 6273 }, "optimeDate" : ISODate("2013-01-16T23:36:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1564, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1551, "optime" : { "t" : 1358379382000, "i" : 4656 }, "optimeDate" : ISODate("2013-01-16T23:36:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:26Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1551, "optime" : { "t" : 1358379385000, "i" : 1641 }, "optimeDate" : ISODate("2013-01-16T23:36:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1551, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1563, "optime" : { "t" : 1358379386000, "i" : 6313 }, "optimeDate" : ISODate("2013-01-16T23:36:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:36:28.285 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:36:28.700 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:36:30.285 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1569, "optime" : { "t" : 1358379390000, "i" : 7371 }, "optimeDate" : ISODate("2013-01-16T23:36:30Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1558, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1558, "optime" : { "t" : 1358379386000, "i" : 6313 }, "optimeDate" : ISODate("2013-01-16T23:36:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:29Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1568, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1555, "optime" : { "t" : 1358379386000, "i" : 6313 }, "optimeDate" : ISODate("2013-01-16T23:36:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1555, "optime" : { "t" : 1358379389000, "i" : 227 }, "optimeDate" : ISODate("2013-01-16T23:36:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1555, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:28Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1567, "optime" : { "t" : 1358379390000, "i" : 7371 }, "optimeDate" : ISODate("2013-01-16T23:36:30Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:36:32.286 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:36:32.901 [conn94] end connection 127.0.0.3:46872 (3 connections now open) m31002| Wed Jan 16 18:36:32.901 [initandlisten] connection accepted from 127.0.0.3:45733 #96 (4 connections now open) m31001| Wed Jan 16 18:36:33.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:36:33.700 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:36:34.286 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:36:34.410 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:460580 230ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1573, "optime" : { "t" : 1358379394000, "i" : 8741 }, "optimeDate" : ISODate("2013-01-16T23:36:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1562, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1562, "optime" : { "t" : 1358379394000, "i" : 340 }, "optimeDate" : ISODate("2013-01-16T23:36:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1572, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1559, "optime" : { "t" : 1358379390000, "i" : 7371 }, "optimeDate" : ISODate("2013-01-16T23:36:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:33Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1559, "optime" : { "t" : 1358379390000, "i" : 7371 }, "optimeDate" : ISODate("2013-01-16T23:36:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1559, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1571, "optime" : { "t" : 1358379394000, "i" : 8201 }, "optimeDate" : ISODate("2013-01-16T23:36:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:36:36.287 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:36:38.287 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:36:38.701 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1577, "optime" : { "t" : 1358379398000, "i" : 8843 }, "optimeDate" : ISODate("2013-01-16T23:36:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1566, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1566, "optime" : { "t" : 1358379394000, "i" : 8741 }, "optimeDate" : ISODate("2013-01-16T23:36:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:37Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1576, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1563, "optime" : { "t" : 1358379398000, "i" : 8285 }, "optimeDate" : ISODate("2013-01-16T23:36:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1563, "optime" : { "t" : 1358379394000, "i" : 8741 }, "optimeDate" : ISODate("2013-01-16T23:36:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1563, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:38Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1575, "optime" : { "t" : 1358379398000, "i" : 8285 }, "optimeDate" : ISODate("2013-01-16T23:36:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:36:40.287 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:36:42.288 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1582, "optime" : { "t" : 1358379403000, "i" : 862 }, "optimeDate" : ISODate("2013-01-16T23:36:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1571, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1571, "optime" : { "t" : 1358379398000, "i" : 8843 }, "optimeDate" : ISODate("2013-01-16T23:36:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:41Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1581, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1568, "optime" : { "t" : 1358379402000, "i" : 7894 }, "optimeDate" : ISODate("2013-01-16T23:36:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1568, "optime" : { "t" : 1358379398000, "i" : 8843 }, "optimeDate" : ISODate("2013-01-16T23:36:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1568, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:42Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1580, "optime" : { "t" : 1358379403000, "i" : 206 }, "optimeDate" : ISODate("2013-01-16T23:36:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:36:43.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:36:43.702 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:36:44.288 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:36:46.288 [conn93] end connection 127.0.0.2:57391 (2 connections now open) m31001| Wed Jan 16 18:36:46.288 [initandlisten] connection accepted from 127.0.0.2:34599 #95 (3 connections now open) m31000| Wed Jan 16 18:36:46.288 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1586, "optime" : { "t" : 1358379407000, "i" : 2158 }, "optimeDate" : ISODate("2013-01-16T23:36:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1575, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1575, "optime" : { "t" : 1358379406000, "i" : 5463 }, "optimeDate" : ISODate("2013-01-16T23:36:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1585, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1572, "optime" : { "t" : 1358379406000, "i" : 7068 }, "optimeDate" : ISODate("2013-01-16T23:36:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1572, "optime" : { "t" : 1358379403000, "i" : 862 }, "optimeDate" : ISODate("2013-01-16T23:36:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1572, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:46Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1584, "optime" : { "t" : 1358379407000, "i" : 2158 }, "optimeDate" : ISODate("2013-01-16T23:36:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:36:48.289 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:36:48.702 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:36:49.756 [conn62] end connection 127.0.0.4:60654 (4 connections now open) m31000| Wed Jan 16 18:36:49.756 [initandlisten] connection accepted from 127.0.0.4:33779 #63 (5 connections now open) m31000| Wed Jan 16 18:36:50.289 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1590, "optime" : { "t" : 1358379411000, "i" : 3141 }, "optimeDate" : ISODate("2013-01-16T23:36:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1579, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1579, "optime" : { "t" : 1358379407000, "i" : 2158 }, "optimeDate" : ISODate("2013-01-16T23:36:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:49Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1589, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1576, "optime" : { "t" : 1358379410000, "i" : 5814 }, "optimeDate" : ISODate("2013-01-16T23:36:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:51Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1576, "optime" : { "t" : 1358379407000, "i" : 2158 }, "optimeDate" : ISODate("2013-01-16T23:36:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1576, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1588, "optime" : { "t" : 1358379411000, "i" : 2972 }, "optimeDate" : ISODate("2013-01-16T23:36:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:36:51.710 [conn95] end connection 127.0.0.2:45611 (3 connections now open) m31002| Wed Jan 16 18:36:52.211 [initandlisten] connection accepted from 127.0.0.2:50305 #97 (4 connections now open) m31000| Wed Jan 16 18:36:52.289 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:36:53.703 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:36:54.290 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1594, "optime" : { "t" : 1358379415000, "i" : 5011 }, "optimeDate" : ISODate("2013-01-16T23:36:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1583, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1583, "optime" : { "t" : 1358379414000, "i" : 2603 }, "optimeDate" : ISODate("2013-01-16T23:36:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1593, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1580, "optime" : { "t" : 1358379414000, "i" : 3841 }, "optimeDate" : ISODate("2013-01-16T23:36:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1580, "optime" : { "t" : 1358379411000, "i" : 3141 }, "optimeDate" : ISODate("2013-01-16T23:36:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1580, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:54Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1592, "optime" : { "t" : 1358379415000, "i" : 5011 }, "optimeDate" : ISODate("2013-01-16T23:36:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:36:56.203 [conn94] end connection 127.0.0.4:59518 (2 connections now open) m31000| Wed Jan 16 18:36:56.290 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:36:56.703 [initandlisten] connection accepted from 127.0.0.4:42326 #96 (3 connections now open) m31000| Wed Jan 16 18:36:58.291 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:36:59.204 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1598, "optime" : { "t" : 1358379419000, "i" : 5941 }, "optimeDate" : ISODate("2013-01-16T23:36:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1587, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1587, "optime" : { "t" : 1358379415000, "i" : 5011 }, "optimeDate" : ISODate("2013-01-16T23:36:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:57Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1597, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1584, "optime" : { "t" : 1358379418000, "i" : 2877 }, "optimeDate" : ISODate("2013-01-16T23:36:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:36:59Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:36:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1584, "optime" : { "t" : 1358379415000, "i" : 5011 }, "optimeDate" : ISODate("2013-01-16T23:36:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1584, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1596, "optime" : { "t" : 1358379419000, "i" : 5941 }, "optimeDate" : ISODate("2013-01-16T23:36:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:37:00.291 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:37:02.291 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:37:02.906 [conn96] end connection 127.0.0.3:45733 (3 connections now open) m31002| Wed Jan 16 18:37:02.906 [initandlisten] connection accepted from 127.0.0.3:58425 #98 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1602, "optime" : { "t" : 1358379423000, "i" : 7185 }, "optimeDate" : ISODate("2013-01-16T23:37:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1591, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1591, "optime" : { "t" : 1358379419000, "i" : 5941 }, "optimeDate" : ISODate("2013-01-16T23:36:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:03Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1601, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1588, "optime" : { "t" : 1358379422000, "i" : 1516 }, "optimeDate" : ISODate("2013-01-16T23:37:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1588, "optime" : { "t" : 1358379423000, "i" : 6339 }, "optimeDate" : ISODate("2013-01-16T23:37:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1588, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:02Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1600, "optime" : { "t" : 1358379423000, "i" : 6965 }, "optimeDate" : ISODate("2013-01-16T23:37:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:37:04.204 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:37:04.292 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:37:05.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:37:06.292 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1606, "optime" : { "t" : 1358379427000, "i" : 8597 }, "optimeDate" : ISODate("2013-01-16T23:37:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1595, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1595, "optime" : { "t" : 1358379427000, "i" : 1932 }, "optimeDate" : ISODate("2013-01-16T23:37:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:07Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1605, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1592, "optime" : { "t" : 1358379426000, "i" : 394 }, "optimeDate" : ISODate("2013-01-16T23:37:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1592, "optime" : { "t" : 1358379427000, "i" : 6870 }, "optimeDate" : ISODate("2013-01-16T23:37:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1592, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:06Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1604, "optime" : { "t" : 1358379427000, "i" : 8597 }, "optimeDate" : ISODate("2013-01-16T23:37:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:37:08.293 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:37:09.205 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:37:10.293 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1611, "optime" : { "t" : 1358379432000, "i" : 981 }, "optimeDate" : ISODate("2013-01-16T23:37:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1600, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1600, "optime" : { "t" : 1358379427000, "i" : 8597 }, "optimeDate" : ISODate("2013-01-16T23:37:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:11Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1610, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1597, "optime" : { "t" : 1358379427000, "i" : 8597 }, "optimeDate" : ISODate("2013-01-16T23:37:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:11Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1597, "optime" : { "t" : 1358379431000, "i" : 6982 }, "optimeDate" : ISODate("2013-01-16T23:37:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1597, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1609, "optime" : { "t" : 1358379432000, "i" : 981 }, "optimeDate" : ISODate("2013-01-16T23:37:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:37:12.293 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:37:14.205 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:37:14.294 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:37:15.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1615, "optime" : { "t" : 1358379436000, "i" : 2476 }, "optimeDate" : ISODate("2013-01-16T23:37:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1604, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1604, "optime" : { "t" : 1358379432000, "i" : 981 }, "optimeDate" : ISODate("2013-01-16T23:37:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:15Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 m31001| Wed Jan 16 18:37:16.294 [conn95] end connection 127.0.0.2:34599 (2 connections now open) m31001| Wed Jan 16 18:37:16.294 [initandlisten] connection accepted from 127.0.0.2:38409 #97 (3 connections now open) m31000| Wed Jan 16 18:37:16.295 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1614, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1601, "optime" : { "t" : 1358379432000, "i" : 981 }, "optimeDate" : ISODate("2013-01-16T23:37:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1601, "optime" : { "t" : 1358379435000, "i" : 5019 }, "optimeDate" : ISODate("2013-01-16T23:37:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1601, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:14Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1613, "optime" : { "t" : 1358379436000, "i" : 2476 }, "optimeDate" : ISODate("2013-01-16T23:37:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:37:18.295 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:37:19.206 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:37:19.762 [conn63] end connection 127.0.0.4:33779 (4 connections now open) m31000| Wed Jan 16 18:37:19.762 [initandlisten] connection accepted from 127.0.0.4:43856 #64 (5 connections now open) m31000| Wed Jan 16 18:37:20.295 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1619, "optime" : { "t" : 1358379440000, "i" : 4430 }, "optimeDate" : ISODate("2013-01-16T23:37:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1608, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1608, "optime" : { "t" : 1358379439000, "i" : 3219 }, "optimeDate" : ISODate("2013-01-16T23:37:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:19Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1618, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1605, "optime" : { "t" : 1358379436000, "i" : 2476 }, "optimeDate" : ISODate("2013-01-16T23:37:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:19Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1605, "optime" : { "t" : 1358379439000, "i" : 3839 }, "optimeDate" : ISODate("2013-01-16T23:37:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1605, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1617, "optime" : { "t" : 1358379440000, "i" : 4430 }, "optimeDate" : ISODate("2013-01-16T23:37:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:37:22.296 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:37:24.207 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:37:24.296 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1623, "optime" : { "t" : 1358379444000, "i" : 4785 }, "optimeDate" : ISODate("2013-01-16T23:37:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1612, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1612, "optime" : { "t" : 1358379440000, "i" : 4430 }, "optimeDate" : ISODate("2013-01-16T23:37:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:23Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:36:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1622, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1609, "optime" : { "t" : 1358379440000, "i" : 4430 }, "optimeDate" : ISODate("2013-01-16T23:37:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:24Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1609, "optime" : { "t" : 1358379443000, "i" : 2712 }, "optimeDate" : ISODate("2013-01-16T23:37:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1609, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1621, "optime" : { "t" : 1358379444000, "i" : 4038 }, "optimeDate" : ISODate("2013-01-16T23:37:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:37:26.296 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:37:28.297 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1627, "optime" : { "t" : 1358379448000, "i" : 5744 }, "optimeDate" : ISODate("2013-01-16T23:37:28Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1616, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1616, "optime" : { "t" : 1358379444000, "i" : 4785 }, "optimeDate" : ISODate("2013-01-16T23:37:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:27Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1626, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1613, "optime" : { "t" : 1358379444000, "i" : 4785 }, "optimeDate" : ISODate("2013-01-16T23:37:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:28Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1613, "optime" : { "t" : 1358379447000, "i" : 2373 }, "optimeDate" : ISODate("2013-01-16T23:37:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1613, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:26Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1625, "optime" : { "t" : 1358379448000, "i" : 5744 }, "optimeDate" : ISODate("2013-01-16T23:37:28Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:37:29.208 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:37:29.716 [conn97] end connection 127.0.0.2:50305 (3 connections now open) m31002| Wed Jan 16 18:37:30.216 [initandlisten] connection accepted from 127.0.0.2:50817 #99 (4 connections now open) m31000| Wed Jan 16 18:37:30.297 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:37:32.297 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1631, "optime" : { "t" : 1358379452000, "i" : 6697 }, "optimeDate" : ISODate("2013-01-16T23:37:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1620, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1620, "optime" : { "t" : 1358379452000, "i" : 5853 }, "optimeDate" : ISODate("2013-01-16T23:37:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1630, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1617, "optime" : { "t" : 1358379448000, "i" : 5744 }, "optimeDate" : ISODate("2013-01-16T23:37:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:31Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1617, "optime" : { "t" : 1358379451000, "i" : 1160 }, "optimeDate" : ISODate("2013-01-16T23:37:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1617, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1629, "optime" : { "t" : 1358379452000, "i" : 6697 }, "optimeDate" : ISODate("2013-01-16T23:37:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:37:32.912 [conn98] end connection 127.0.0.3:58425 (3 connections now open) m31002| Wed Jan 16 18:37:32.912 [initandlisten] connection accepted from 127.0.0.3:59979 #100 (4 connections now open) m31001| Wed Jan 16 18:37:34.208 [conn96] end connection 127.0.0.4:42326 (2 connections now open) m31000| Wed Jan 16 18:37:34.298 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:37:34.708 [initandlisten] connection accepted from 127.0.0.4:54211 #98 (3 connections now open) m31002| Wed Jan 16 18:37:34.708 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:37:36.298 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1635, "optime" : { "t" : 1358379456000, "i" : 8206 }, "optimeDate" : ISODate("2013-01-16T23:37:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1624, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1624, "optime" : { "t" : 1358379452000, "i" : 6697 }, "optimeDate" : ISODate("2013-01-16T23:37:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:35Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1634, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1621, "optime" : { "t" : 1358379456000, "i" : 7746 }, "optimeDate" : ISODate("2013-01-16T23:37:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1621, "optime" : { "t" : 1358379452000, "i" : 6697 }, "optimeDate" : ISODate("2013-01-16T23:37:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1621, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:36Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1633, "optime" : { "t" : 1358379456000, "i" : 8206 }, "optimeDate" : ISODate("2013-01-16T23:37:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:37:37.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:37:38.298 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:37:39.709 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:37:40.299 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1640, "optime" : { "t" : 1358379461000, "i" : 394 }, "optimeDate" : ISODate("2013-01-16T23:37:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1629, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1629, "optime" : { "t" : 1358379460000, "i" : 2154 }, "optimeDate" : ISODate("2013-01-16T23:37:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1639, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1626, "optime" : { "t" : 1358379460000, "i" : 8690 }, "optimeDate" : ISODate("2013-01-16T23:37:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:41Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1626, "optime" : { "t" : 1358379456000, "i" : 8206 }, "optimeDate" : ISODate("2013-01-16T23:37:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1626, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:40Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1638, "optime" : { "t" : 1358379460000, "i" : 8978 }, "optimeDate" : ISODate("2013-01-16T23:37:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:37:42.299 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:37:44.299 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:37:44.380 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:485758 242ms m31002| Wed Jan 16 18:37:44.710 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1644, "optime" : { "t" : 1358379465000, "i" : 217 }, "optimeDate" : ISODate("2013-01-16T23:37:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1633, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1633, "optime" : { "t" : 1358379461000, "i" : 394 }, "optimeDate" : ISODate("2013-01-16T23:37:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:43Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1643, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1630, "optime" : { "t" : 1358379464000, "i" : 8735 }, "optimeDate" : ISODate("2013-01-16T23:37:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1630, "optime" : { "t" : 1358379461000, "i" : 394 }, "optimeDate" : ISODate("2013-01-16T23:37:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1630, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:44Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1642, "optime" : { "t" : 1358379465000, "i" : 217 }, "optimeDate" : ISODate("2013-01-16T23:37:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:37:46.300 [conn97] end connection 127.0.0.2:38409 (2 connections now open) m31001| Wed Jan 16 18:37:46.300 [initandlisten] connection accepted from 127.0.0.2:38080 #99 (3 connections now open) m31000| Wed Jan 16 18:37:46.300 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:37:47.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:37:48.300 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1648, "optime" : { "t" : 1358379469000, "i" : 2429 }, "optimeDate" : ISODate("2013-01-16T23:37:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1637, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1637, "optime" : { "t" : 1358379465000, "i" : 217 }, "optimeDate" : ISODate("2013-01-16T23:37:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:47Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1647, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1634, "optime" : { "t" : 1358379468000, "i" : 6775 }, "optimeDate" : ISODate("2013-01-16T23:37:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1634, "optime" : { "t" : 1358379465000, "i" : 217 }, "optimeDate" : ISODate("2013-01-16T23:37:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1634, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:48Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1646, "optime" : { "t" : 1358379469000, "i" : 2429 }, "optimeDate" : ISODate("2013-01-16T23:37:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:37:49.710 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:37:49.767 [conn64] end connection 127.0.0.4:43856 (4 connections now open) m31000| Wed Jan 16 18:37:49.767 [initandlisten] connection accepted from 127.0.0.4:35285 #65 (5 connections now open) m31000| Wed Jan 16 18:37:50.301 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:37:52.301 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1652, "optime" : { "t" : 1358379473000, "i" : 4494 }, "optimeDate" : ISODate("2013-01-16T23:37:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1641, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1641, "optime" : { "t" : 1358379472000, "i" : 3369 }, "optimeDate" : ISODate("2013-01-16T23:37:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1651, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1638, "optime" : { "t" : 1358379472000, "i" : 5001 }, "optimeDate" : ISODate("2013-01-16T23:37:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1638, "optime" : { "t" : 1358379469000, "i" : 2429 }, "optimeDate" : ISODate("2013-01-16T23:37:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1638, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:52Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1650, "optime" : { "t" : 1358379473000, "i" : 4494 }, "optimeDate" : ISODate("2013-01-16T23:37:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:37:54.301 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:37:54.711 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:37:56.302 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1656, "optime" : { "t" : 1358379477000, "i" : 6191 }, "optimeDate" : ISODate("2013-01-16T23:37:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1645, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1645, "optime" : { "t" : 1358379477000, "i" : 5936 }, "optimeDate" : ISODate("2013-01-16T23:37:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1655, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1642, "optime" : { "t" : 1358379476000, "i" : 3091 }, "optimeDate" : ISODate("2013-01-16T23:37:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:57Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:37:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1642, "optime" : { "t" : 1358379473000, "i" : 4494 }, "optimeDate" : ISODate("2013-01-16T23:37:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:37:57Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1642, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1654, "optime" : { "t" : 1358379477000, "i" : 6191 }, "optimeDate" : ISODate("2013-01-16T23:37:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:37:58.302 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:37:59.712 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:38:00.303 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1660, "optime" : { "t" : 1358379481000, "i" : 7835 }, "optimeDate" : ISODate("2013-01-16T23:38:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1649, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1649, "optime" : { "t" : 1358379477000, "i" : 6191 }, "optimeDate" : ISODate("2013-01-16T23:37:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:01Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1659, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1646, "optime" : { "t" : 1358379480000, "i" : 1441 }, "optimeDate" : ISODate("2013-01-16T23:38:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1646, "optime" : { "t" : 1358379481000, "i" : 6749 }, "optimeDate" : ISODate("2013-01-16T23:38:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1646, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1658, "optime" : { "t" : 1358379481000, "i" : 7562 }, "optimeDate" : ISODate("2013-01-16T23:38:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:38:02.303 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:38:02.917 [conn100] end connection 127.0.0.3:59979 (3 connections now open) m31002| Wed Jan 16 18:38:02.917 [initandlisten] connection accepted from 127.0.0.3:36176 #101 (4 connections now open) m31000| Wed Jan 16 18:38:04.304 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:38:04.713 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1665, "optime" : { "t" : 1358379486000, "i" : 42 }, "optimeDate" : ISODate("2013-01-16T23:38:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1654, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1654, "optime" : { "t" : 1358379485000, "i" : 1797 }, "optimeDate" : ISODate("2013-01-16T23:38:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:05Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1664, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1651, "optime" : { "t" : 1358379484000, "i" : 459 }, "optimeDate" : ISODate("2013-01-16T23:38:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1651, "optime" : { "t" : 1358379485000, "i" : 6545 }, "optimeDate" : ISODate("2013-01-16T23:38:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1651, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:04Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1663, "optime" : { "t" : 1358379485000, "i" : 8508 }, "optimeDate" : ISODate("2013-01-16T23:38:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:38:06.304 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:38:07.721 [conn99] end connection 127.0.0.2:50817 (3 connections now open) m31002| Wed Jan 16 18:38:08.221 [initandlisten] connection accepted from 127.0.0.2:58323 #102 (4 connections now open) m31000| Wed Jan 16 18:38:08.304 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:38:09.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:38:09.713 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1669, "optime" : { "t" : 1358379490000, "i" : 1139 }, "optimeDate" : ISODate("2013-01-16T23:38:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1658, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1658, "optime" : { "t" : 1358379486000, "i" : 42 }, "optimeDate" : ISODate("2013-01-16T23:38:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:09Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1668, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1655, "optime" : { "t" : 1358379486000, "i" : 42 }, "optimeDate" : ISODate("2013-01-16T23:38:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:09Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1655, "optime" : { "t" : 1358379489000, "i" : 6695 }, "optimeDate" : ISODate("2013-01-16T23:38:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1655, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1667, "optime" : { "t" : 1358379490000, "i" : 1139 }, "optimeDate" : ISODate("2013-01-16T23:38:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:38:10.305 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:38:12.213 [conn98] end connection 127.0.0.4:54211 (2 connections now open) m31000| Wed Jan 16 18:38:12.305 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:38:12.714 [initandlisten] connection accepted from 127.0.0.4:47517 #100 (3 connections now open) STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:38:14.306 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:38:14.365 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:413995 207ms { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1673, "optime" : { "t" : 1358379494000, "i" : 2410 }, "optimeDate" : ISODate("2013-01-16T23:38:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1662, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1662, "optime" : { "t" : 1358379493000, "i" : 308 }, "optimeDate" : ISODate("2013-01-16T23:38:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:13Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1672, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1659, "optime" : { "t" : 1358379490000, "i" : 1139 }, "optimeDate" : ISODate("2013-01-16T23:38:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1659, "optime" : { "t" : 1358379493000, "i" : 5302 }, "optimeDate" : ISODate("2013-01-16T23:38:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1659, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:12Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1671, "optime" : { "t" : 1358379494000, "i" : 2303 }, "optimeDate" : ISODate("2013-01-16T23:38:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:38:15.214 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:38:16.306 [conn99] end connection 127.0.0.2:38080 (2 connections now open) m31001| Wed Jan 16 18:38:16.306 [initandlisten] connection accepted from 127.0.0.2:52945 #101 (3 connections now open) m31000| Wed Jan 16 18:38:16.306 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:38:18.306 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:38:18.479 [journal] old journal file will be removed: /data/db/testReplSet-0/journal/j._1 STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:38:18.548 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:296171 148ms { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1677, "optime" : { "t" : 1358379498000, "i" : 4796 }, "optimeDate" : ISODate("2013-01-16T23:38:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1666, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1666, "optime" : { "t" : 1358379498000, "i" : 1830 }, "optimeDate" : ISODate("2013-01-16T23:38:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1676, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1663, "optime" : { "t" : 1358379494000, "i" : 2410 }, "optimeDate" : ISODate("2013-01-16T23:38:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:17Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1663, "optime" : { "t" : 1358379497000, "i" : 2834 }, "optimeDate" : ISODate("2013-01-16T23:38:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1663, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1675, "optime" : { "t" : 1358379498000, "i" : 3369 }, "optimeDate" : ISODate("2013-01-16T23:38:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:38:19.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:38:19.772 [conn65] end connection 127.0.0.4:35285 (4 connections now open) m31000| Wed Jan 16 18:38:19.773 [initandlisten] connection accepted from 127.0.0.4:60845 #66 (5 connections now open) m31002| Wed Jan 16 18:38:20.215 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:38:20.307 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:38:22.307 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1681, "optime" : { "t" : 1358379502000, "i" : 6319 }, "optimeDate" : ISODate("2013-01-16T23:38:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1670, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1670, "optime" : { "t" : 1358379498000, "i" : 4796 }, "optimeDate" : ISODate("2013-01-16T23:38:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:21Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1680, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1667, "optime" : { "t" : 1358379498000, "i" : 4796 }, "optimeDate" : ISODate("2013-01-16T23:38:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1667, "optime" : { "t" : 1358379501000, "i" : 1282 }, "optimeDate" : ISODate("2013-01-16T23:38:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1667, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:20Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1679, "optime" : { "t" : 1358379502000, "i" : 6177 }, "optimeDate" : ISODate("2013-01-16T23:38:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:38:24.308 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:38:25.215 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:38:26.308 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1685, "optime" : { "t" : 1358379506000, "i" : 7553 }, "optimeDate" : ISODate("2013-01-16T23:38:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1674, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1674, "optime" : { "t" : 1358379505000, "i" : 25 }, "optimeDate" : ISODate("2013-01-16T23:38:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:25Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1684, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1671, "optime" : { "t" : 1358379502000, "i" : 6319 }, "optimeDate" : ISODate("2013-01-16T23:38:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:25Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1671, "optime" : { "t" : 1358379505000, "i" : 361 }, "optimeDate" : ISODate("2013-01-16T23:38:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1671, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1683, "optime" : { "t" : 1358379506000, "i" : 7553 }, "optimeDate" : ISODate("2013-01-16T23:38:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:38:28.308 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:38:29.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:38:30.216 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:38:30.309 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1690, "optime" : { "t" : 1358379511000, "i" : 759 }, "optimeDate" : ISODate("2013-01-16T23:38:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1679, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1679, "optime" : { "t" : 1358379510000, "i" : 5728 }, "optimeDate" : ISODate("2013-01-16T23:38:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1689, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1676, "optime" : { "t" : 1358379510000, "i" : 7511 }, "optimeDate" : ISODate("2013-01-16T23:38:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1676, "optime" : { "t" : 1358379506000, "i" : 7553 }, "optimeDate" : ISODate("2013-01-16T23:38:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1676, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:30Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1688, "optime" : { "t" : 1358379511000, "i" : 759 }, "optimeDate" : ISODate("2013-01-16T23:38:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:38:32.309 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:38:32.923 [conn101] end connection 127.0.0.3:36176 (3 connections now open) m31002| Wed Jan 16 18:38:32.923 [initandlisten] connection accepted from 127.0.0.3:54030 #103 (4 connections now open) m31000| Wed Jan 16 18:38:34.309 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:38:34.313 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:277710 138ms m31002| Wed Jan 16 18:38:35.217 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1694, "optime" : { "t" : 1358379515000, "i" : 1819 }, "optimeDate" : ISODate("2013-01-16T23:38:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1683, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1683, "optime" : { "t" : 1358379511000, "i" : 759 }, "optimeDate" : ISODate("2013-01-16T23:38:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:33Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1693, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1680, "optime" : { "t" : 1358379514000, "i" : 7060 }, "optimeDate" : ISODate("2013-01-16T23:38:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:35Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1680, "optime" : { "t" : 1358379511000, "i" : 759 }, "optimeDate" : ISODate("2013-01-16T23:38:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1680, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1692, "optime" : { "t" : 1358379515000, "i" : 1698 }, "optimeDate" : ISODate("2013-01-16T23:38:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:38:36.310 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:38:38.310 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1698, "optime" : { "t" : 1358379519000, "i" : 4149 }, "optimeDate" : ISODate("2013-01-16T23:38:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1687, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1687, "optime" : { "t" : 1358379515000, "i" : 1819 }, "optimeDate" : ISODate("2013-01-16T23:38:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:37:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1697, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1684, "optime" : { "t" : 1358379518000, "i" : 5502 }, "optimeDate" : ISODate("2013-01-16T23:38:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1684, "optime" : { "t" : 1358379515000, "i" : 1819 }, "optimeDate" : ISODate("2013-01-16T23:38:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1684, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:38Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1696, "optime" : { "t" : 1358379519000, "i" : 4149 }, "optimeDate" : ISODate("2013-01-16T23:38:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:38:40.217 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:38:40.310 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:38:42.311 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1702, "optime" : { "t" : 1358379523000, "i" : 5081 }, "optimeDate" : ISODate("2013-01-16T23:38:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1691, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1691, "optime" : { "t" : 1358379523000, "i" : 2284 }, "optimeDate" : ISODate("2013-01-16T23:38:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1701, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1688, "optime" : { "t" : 1358379522000, "i" : 4635 }, "optimeDate" : ISODate("2013-01-16T23:38:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1688, "optime" : { "t" : 1358379519000, "i" : 4149 }, "optimeDate" : ISODate("2013-01-16T23:38:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:43Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1688, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:42Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1700, "optime" : { "t" : 1358379523000, "i" : 4513 }, "optimeDate" : ISODate("2013-01-16T23:38:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:38:44.311 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:38:45.218 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:38:45.727 [conn102] end connection 127.0.0.2:58323 (3 connections now open) m31002| Wed Jan 16 18:38:46.227 [initandlisten] connection accepted from 127.0.0.2:58599 #104 (4 connections now open) m31001| Wed Jan 16 18:38:46.311 [conn101] end connection 127.0.0.2:52945 (2 connections now open) m31001| Wed Jan 16 18:38:46.312 [initandlisten] connection accepted from 127.0.0.2:59101 #102 (3 connections now open) m31000| Wed Jan 16 18:38:46.312 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1706, "optime" : { "t" : 1358379527000, "i" : 5897 }, "optimeDate" : ISODate("2013-01-16T23:38:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1695, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1695, "optime" : { "t" : 1358379523000, "i" : 5081 }, "optimeDate" : ISODate("2013-01-16T23:38:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1705, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1692, "optime" : { "t" : 1358379526000, "i" : 2948 }, "optimeDate" : ISODate("2013-01-16T23:38:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:47Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1692, "optime" : { "t" : 1358379523000, "i" : 5081 }, "optimeDate" : ISODate("2013-01-16T23:38:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1692, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1704, "optime" : { "t" : 1358379527000, "i" : 5897 }, "optimeDate" : ISODate("2013-01-16T23:38:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:38:48.312 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:38:49.778 [conn66] end connection 127.0.0.4:60845 (4 connections now open) m31000| Wed Jan 16 18:38:49.778 [initandlisten] connection accepted from 127.0.0.4:36407 #67 (5 connections now open) m31001| Wed Jan 16 18:38:50.219 [conn100] end connection 127.0.0.4:47517 (2 connections now open) m31000| Wed Jan 16 18:38:50.313 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:38:50.719 [initandlisten] connection accepted from 127.0.0.4:33266 #103 (3 connections now open) m31002| Wed Jan 16 18:38:50.719 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:38:51.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:38:51.646 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:273192 136ms m31002| Wed Jan 16 18:38:51.731 [journal] old journal file will be removed: /data/db/testReplSet-2/journal/j._1 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1710, "optime" : { "t" : 1358379531000, "i" : 7746 }, "optimeDate" : ISODate("2013-01-16T23:38:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1699, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1699, "optime" : { "t" : 1358379531000, "i" : 1901 }, "optimeDate" : ISODate("2013-01-16T23:38:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:51Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1709, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1696, "optime" : { "t" : 1358379530000, "i" : 1485 }, "optimeDate" : ISODate("2013-01-16T23:38:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1696, "optime" : { "t" : 1358379531000, "i" : 6727 }, "optimeDate" : ISODate("2013-01-16T23:38:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1696, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:50Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1708, "optime" : { "t" : 1358379531000, "i" : 4356 }, "optimeDate" : ISODate("2013-01-16T23:38:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:38:52.313 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:38:54.314 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:38:55.720 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1714, "optime" : { "t" : 1358379535000, "i" : 8721 }, "optimeDate" : ISODate("2013-01-16T23:38:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1703, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1703, "optime" : { "t" : 1358379531000, "i" : 7746 }, "optimeDate" : ISODate("2013-01-16T23:38:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:55Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1713, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1700, "optime" : { "t" : 1358379534000, "i" : 106 }, "optimeDate" : ISODate("2013-01-16T23:38:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:55Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:38:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1700, "optime" : { "t" : 1358379535000, "i" : 6874 }, "optimeDate" : ISODate("2013-01-16T23:38:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1700, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1712, "optime" : { "t" : 1358379535000, "i" : 8710 }, "optimeDate" : ISODate("2013-01-16T23:38:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:38:56.314 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:38:58.314 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1719, "optime" : { "t" : 1358379540000, "i" : 1607 }, "optimeDate" : ISODate("2013-01-16T23:39:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1708, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1708, "optime" : { "t" : 1358379535000, "i" : 8721 }, "optimeDate" : ISODate("2013-01-16T23:38:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:59Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1718, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1705, "optime" : { "t" : 1358379535000, "i" : 8721 }, "optimeDate" : ISODate("2013-01-16T23:38:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1705, "optime" : { "t" : 1358379539000, "i" : 6626 }, "optimeDate" : ISODate("2013-01-16T23:38:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1705, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:38:58Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1717, "optime" : { "t" : 1358379540000, "i" : 1607 }, "optimeDate" : ISODate("2013-01-16T23:39:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:39:00.315 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:39:00.720 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:39:01.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:39:02.315 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:39:02.928 [conn103] end connection 127.0.0.3:54030 (3 connections now open) m31002| Wed Jan 16 18:39:02.928 [initandlisten] connection accepted from 127.0.0.3:57243 #105 (4 connections now open) m31000| Wed Jan 16 18:39:04.315 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1723, "optime" : { "t" : 1358379544000, "i" : 2971 }, "optimeDate" : ISODate("2013-01-16T23:39:04Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1712, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1712, "optime" : { "t" : 1358379543000, "i" : 4396 }, "optimeDate" : ISODate("2013-01-16T23:39:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:03Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1722, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1709, "optime" : { "t" : 1358379540000, "i" : 1607 }, "optimeDate" : ISODate("2013-01-16T23:39:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:03Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1709, "optime" : { "t" : 1358379543000, "i" : 4879 }, "optimeDate" : ISODate("2013-01-16T23:39:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1709, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1721, "optime" : { "t" : 1358379544000, "i" : 2971 }, "optimeDate" : ISODate("2013-01-16T23:39:04Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:39:05.721 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:39:06.316 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:39:08.316 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1727, "optime" : { "t" : 1358379548000, "i" : 4459 }, "optimeDate" : ISODate("2013-01-16T23:39:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1716, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1716, "optime" : { "t" : 1358379544000, "i" : 2971 }, "optimeDate" : ISODate("2013-01-16T23:39:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:07Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:38:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1726, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1713, "optime" : { "t" : 1358379544000, "i" : 2971 }, "optimeDate" : ISODate("2013-01-16T23:39:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:08Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1713, "optime" : { "t" : 1358379547000, "i" : 3124 }, "optimeDate" : ISODate("2013-01-16T23:39:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1713, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1725, "optime" : { "t" : 1358379548000, "i" : 4459 }, "optimeDate" : ISODate("2013-01-16T23:39:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:39:10.316 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:39:10.722 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:39:12.317 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1731, "optime" : { "t" : 1358379552000, "i" : 5479 }, "optimeDate" : ISODate("2013-01-16T23:39:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1720, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1720, "optime" : { "t" : 1358379548000, "i" : 4459 }, "optimeDate" : ISODate("2013-01-16T23:39:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:11Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1730, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1717, "optime" : { "t" : 1358379548000, "i" : 4459 }, "optimeDate" : ISODate("2013-01-16T23:39:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1717, "optime" : { "t" : 1358379551000, "i" : 2192 }, "optimeDate" : ISODate("2013-01-16T23:39:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1717, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:10Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1729, "optime" : { "t" : 1358379552000, "i" : 4838 }, "optimeDate" : ISODate("2013-01-16T23:39:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:39:14.317 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:39:15.722 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:39:16.317 [conn102] end connection 127.0.0.2:59101 (2 connections now open) m31001| Wed Jan 16 18:39:16.317 [initandlisten] connection accepted from 127.0.0.2:32878 #104 (3 connections now open) m31000| Wed Jan 16 18:39:16.318 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1735, "optime" : { "t" : 1358379556000, "i" : 7196 }, "optimeDate" : ISODate("2013-01-16T23:39:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1724, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1724, "optime" : { "t" : 1358379556000, "i" : 2186 }, "optimeDate" : ISODate("2013-01-16T23:39:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1734, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1721, "optime" : { "t" : 1358379552000, "i" : 5479 }, "optimeDate" : ISODate("2013-01-16T23:39:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:15Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1721, "optime" : { "t" : 1358379555000, "i" : 1236 }, "optimeDate" : ISODate("2013-01-16T23:39:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1721, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1733, "optime" : { "t" : 1358379556000, "i" : 7196 }, "optimeDate" : ISODate("2013-01-16T23:39:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:39:18.318 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:39:19.783 [conn67] end connection 127.0.0.4:36407 (4 connections now open) m31000| Wed Jan 16 18:39:19.783 [initandlisten] connection accepted from 127.0.0.4:46666 #68 (5 connections now open) m31000| Wed Jan 16 18:39:20.318 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:39:20.723 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1739, "optime" : { "t" : 1358379560000, "i" : 8601 }, "optimeDate" : ISODate("2013-01-16T23:39:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1728, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1728, "optime" : { "t" : 1358379556000, "i" : 7196 }, "optimeDate" : ISODate("2013-01-16T23:39:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:19Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1738, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1725, "optime" : { "t" : 1358379560000, "i" : 7793 }, "optimeDate" : ISODate("2013-01-16T23:39:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1725, "optime" : { "t" : 1358379556000, "i" : 7196 }, "optimeDate" : ISODate("2013-01-16T23:39:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1725, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:20Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1737, "optime" : { "t" : 1358379560000, "i" : 7794 }, "optimeDate" : ISODate("2013-01-16T23:39:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:39:22.319 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:39:23.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:39:23.732 [conn104] end connection 127.0.0.2:58599 (3 connections now open) m31002| Wed Jan 16 18:39:24.232 [initandlisten] connection accepted from 127.0.0.2:36948 #106 (4 connections now open) m31000| Wed Jan 16 18:39:24.319 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:39:24.410 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:504804 252ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1743, "optime" : { "t" : 1358379564000, "i" : 9605 }, "optimeDate" : ISODate("2013-01-16T23:39:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1732, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1732, "optime" : { "t" : 1358379564000, "i" : 1514 }, "optimeDate" : ISODate("2013-01-16T23:39:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1742, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1729, "optime" : { "t" : 1358379564000, "i" : 8803 }, "optimeDate" : ISODate("2013-01-16T23:39:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1729, "optime" : { "t" : 1358379560000, "i" : 8601 }, "optimeDate" : ISODate("2013-01-16T23:39:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1729, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:24Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1741, "optime" : { "t" : 1358379564000, "i" : 9515 }, "optimeDate" : ISODate("2013-01-16T23:39:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:39:25.724 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:39:26.319 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:39:28.224 [conn103] end connection 127.0.0.4:33266 (2 connections now open) m31000| Wed Jan 16 18:39:28.320 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:39:28.724 [initandlisten] connection accepted from 127.0.0.4:60727 #105 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1748, "optime" : { "t" : 1358379569000, "i" : 1564 }, "optimeDate" : ISODate("2013-01-16T23:39:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1737, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1737, "optime" : { "t" : 1358379564000, "i" : 9605 }, "optimeDate" : ISODate("2013-01-16T23:39:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:27Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1747, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1734, "optime" : { "t" : 1358379568000, "i" : 7761 }, "optimeDate" : ISODate("2013-01-16T23:39:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1734, "optime" : { "t" : 1358379564000, "i" : 9605 }, "optimeDate" : ISODate("2013-01-16T23:39:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1734, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:28Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1746, "optime" : { "t" : 1358379569000, "i" : 1564 }, "optimeDate" : ISODate("2013-01-16T23:39:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:39:30.320 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:39:31.224 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:39:32.320 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:39:32.933 [conn105] end connection 127.0.0.3:57243 (3 connections now open) m31002| Wed Jan 16 18:39:32.933 [initandlisten] connection accepted from 127.0.0.3:48906 #107 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1752, "optime" : { "t" : 1358379573000, "i" : 3402 }, "optimeDate" : ISODate("2013-01-16T23:39:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1741, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1741, "optime" : { "t" : 1358379569000, "i" : 1564 }, "optimeDate" : ISODate("2013-01-16T23:39:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:31Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1751, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1738, "optime" : { "t" : 1358379572000, "i" : 6209 }, "optimeDate" : ISODate("2013-01-16T23:39:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1738, "optime" : { "t" : 1358379569000, "i" : 1564 }, "optimeDate" : ISODate("2013-01-16T23:39:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1738, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:32Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1750, "optime" : { "t" : 1358379573000, "i" : 3402 }, "optimeDate" : ISODate("2013-01-16T23:39:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:39:33.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:39:34.321 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:39:36.225 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:39:36.321 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1756, "optime" : { "t" : 1358379577000, "i" : 4477 }, "optimeDate" : ISODate("2013-01-16T23:39:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1745, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1745, "optime" : { "t" : 1358379576000, "i" : 2972 }, "optimeDate" : ISODate("2013-01-16T23:39:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1755, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1742, "optime" : { "t" : 1358379576000, "i" : 5115 }, "optimeDate" : ISODate("2013-01-16T23:39:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1742, "optime" : { "t" : 1358379573000, "i" : 3402 }, "optimeDate" : ISODate("2013-01-16T23:39:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1742, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:36Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1754, "optime" : { "t" : 1358379577000, "i" : 4477 }, "optimeDate" : ISODate("2013-01-16T23:39:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:39:38.322 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:39:40.322 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:39:41.226 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1760, "optime" : { "t" : 1358379581000, "i" : 6520 }, "optimeDate" : ISODate("2013-01-16T23:39:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1749, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1749, "optime" : { "t" : 1358379581000, "i" : 5964 }, "optimeDate" : ISODate("2013-01-16T23:39:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1759, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1746, "optime" : { "t" : 1358379580000, "i" : 2890 }, "optimeDate" : ISODate("2013-01-16T23:39:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:41Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:41Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1746, "optime" : { "t" : 1358379577000, "i" : 4477 }, "optimeDate" : ISODate("2013-01-16T23:39:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:41Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1746, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1758, "optime" : { "t" : 1358379581000, "i" : 6512 }, "optimeDate" : ISODate("2013-01-16T23:39:41Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:39:42.322 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:39:44.323 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1764, "optime" : { "t" : 1358379585000, "i" : 7798 }, "optimeDate" : ISODate("2013-01-16T23:39:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1753, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1753, "optime" : { "t" : 1358379581000, "i" : 6520 }, "optimeDate" : ISODate("2013-01-16T23:39:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:45Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1763, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1750, "optime" : { "t" : 1358379584000, "i" : 1315 }, "optimeDate" : ISODate("2013-01-16T23:39:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1750, "optime" : { "t" : 1358379585000, "i" : 7298 }, "optimeDate" : ISODate("2013-01-16T23:39:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1750, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:44Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1762, "optime" : { "t" : 1358379585000, "i" : 7798 }, "optimeDate" : ISODate("2013-01-16T23:39:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:39:46.226 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:39:46.323 [conn104] end connection 127.0.0.2:32878 (2 connections now open) m31001| Wed Jan 16 18:39:46.323 [initandlisten] connection accepted from 127.0.0.2:55440 #106 (3 connections now open) m31000| Wed Jan 16 18:39:46.324 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:39:48.324 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:39:49.789 [conn68] end connection 127.0.0.4:46666 (4 connections now open) m31000| Wed Jan 16 18:39:49.789 [initandlisten] connection accepted from 127.0.0.4:52042 #69 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1768, "optime" : { "t" : 1358379589000, "i" : 8822 }, "optimeDate" : ISODate("2013-01-16T23:39:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1757, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1757, "optime" : { "t" : 1358379589000, "i" : 1970 }, "optimeDate" : ISODate("2013-01-16T23:39:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:49Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1767, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1754, "optime" : { "t" : 1358379588000, "i" : 117 }, "optimeDate" : ISODate("2013-01-16T23:39:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1754, "optime" : { "t" : 1358379589000, "i" : 6913 }, "optimeDate" : ISODate("2013-01-16T23:39:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1754, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:48Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1766, "optime" : { "t" : 1358379589000, "i" : 8822 }, "optimeDate" : ISODate("2013-01-16T23:39:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:39:50.324 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:39:51.227 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:39:52.325 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1773, "optime" : { "t" : 1358379594000, "i" : 1439 }, "optimeDate" : ISODate("2013-01-16T23:39:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1762, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1762, "optime" : { "t" : 1358379589000, "i" : 8822 }, "optimeDate" : ISODate("2013-01-16T23:39:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:53Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1772, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1759, "optime" : { "t" : 1358379589000, "i" : 8822 }, "optimeDate" : ISODate("2013-01-16T23:39:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:53Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1759, "optime" : { "t" : 1358379593000, "i" : 6418 }, "optimeDate" : ISODate("2013-01-16T23:39:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1759, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1771, "optime" : { "t" : 1358379594000, "i" : 1439 }, "optimeDate" : ISODate("2013-01-16T23:39:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:39:54.325 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:39:55.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:39:56.228 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:39:56.326 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:39:58.326 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1777, "optime" : { "t" : 1358379598000, "i" : 3159 }, "optimeDate" : ISODate("2013-01-16T23:39:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1766, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1766, "optime" : { "t" : 1358379594000, "i" : 1439 }, "optimeDate" : ISODate("2013-01-16T23:39:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:57Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1776, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1763, "optime" : { "t" : 1358379594000, "i" : 1439 }, "optimeDate" : ISODate("2013-01-16T23:39:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:39:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1763, "optime" : { "t" : 1358379597000, "i" : 5068 }, "optimeDate" : ISODate("2013-01-16T23:39:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1763, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:39:56Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1775, "optime" : { "t" : 1358379598000, "i" : 2929 }, "optimeDate" : ISODate("2013-01-16T23:39:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:40:00.326 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:40:01.228 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:40:01.737 [conn106] end connection 127.0.0.2:36948 (3 connections now open) m31002| Wed Jan 16 18:40:02.237 [initandlisten] connection accepted from 127.0.0.2:45193 #108 (4 connections now open) m31000| Wed Jan 16 18:40:02.327 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1781, "optime" : { "t" : 1358379602000, "i" : 5028 }, "optimeDate" : ISODate("2013-01-16T23:40:02Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1770, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1770, "optime" : { "t" : 1358379602000, "i" : 2038 }, "optimeDate" : ISODate("2013-01-16T23:40:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1780, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1767, "optime" : { "t" : 1358379598000, "i" : 3159 }, "optimeDate" : ISODate("2013-01-16T23:39:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:01Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1767, "optime" : { "t" : 1358379601000, "i" : 2832 }, "optimeDate" : ISODate("2013-01-16T23:40:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1767, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1779, "optime" : { "t" : 1358379602000, "i" : 5028 }, "optimeDate" : ISODate("2013-01-16T23:40:02Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:40:02.938 [conn107] end connection 127.0.0.3:48906 (3 connections now open) m31002| Wed Jan 16 18:40:02.938 [initandlisten] connection accepted from 127.0.0.3:45290 #109 (4 connections now open) m31000| Wed Jan 16 18:40:04.327 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:40:05.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 18:40:06.229 [conn105] end connection 127.0.0.4:60727 (2 connections now open) m31000| Wed Jan 16 18:40:06.327 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1785, "optime" : { "t" : 1358379606000, "i" : 5872 }, "optimeDate" : ISODate("2013-01-16T23:40:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1774, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1774, "optime" : { "t" : 1358379602000, "i" : 5028 }, "optimeDate" : ISODate("2013-01-16T23:40:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:05Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1784, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1771, "optime" : { "t" : 1358379602000, "i" : 5028 }, "optimeDate" : ISODate("2013-01-16T23:40:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1771, "optime" : { "t" : 1358379605000, "i" : 2164 }, "optimeDate" : ISODate("2013-01-16T23:40:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1771, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:04Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1783, "optime" : { "t" : 1358379606000, "i" : 4833 }, "optimeDate" : ISODate("2013-01-16T23:40:06Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:40:06.729 [initandlisten] connection accepted from 127.0.0.4:39278 #107 (3 connections now open) m31002| Wed Jan 16 18:40:06.729 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:40:08.328 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:40:10.328 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1789, "optime" : { "t" : 1358379610000, "i" : 7762 }, "optimeDate" : ISODate("2013-01-16T23:40:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1778, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1778, "optime" : { "t" : 1358379609000, "i" : 326 }, "optimeDate" : ISODate("2013-01-16T23:40:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:09Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1788, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1775, "optime" : { "t" : 1358379606000, "i" : 5872 }, "optimeDate" : ISODate("2013-01-16T23:40:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:09Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1775, "optime" : { "t" : 1358379609000, "i" : 654 }, "optimeDate" : ISODate("2013-01-16T23:40:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1775, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1787, "optime" : { "t" : 1358379610000, "i" : 7762 }, "optimeDate" : ISODate("2013-01-16T23:40:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:40:11.730 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:40:12.328 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:40:14.329 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1793, "optime" : { "t" : 1358379614000, "i" : 8701 }, "optimeDate" : ISODate("2013-01-16T23:40:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1782, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1782, "optime" : { "t" : 1358379614000, "i" : 6730 }, "optimeDate" : ISODate("2013-01-16T23:40:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:39:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1792, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1779, "optime" : { "t" : 1358379614000, "i" : 8282 }, "optimeDate" : ISODate("2013-01-16T23:40:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1779, "optime" : { "t" : 1358379610000, "i" : 7762 }, "optimeDate" : ISODate("2013-01-16T23:40:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1779, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:14Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1791, "optime" : { "t" : 1358379614000, "i" : 8701 }, "optimeDate" : ISODate("2013-01-16T23:40:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:40:16.329 [conn106] end connection 127.0.0.2:55440 (2 connections now open) m31001| Wed Jan 16 18:40:16.329 [initandlisten] connection accepted from 127.0.0.2:49368 #108 (3 connections now open) m31000| Wed Jan 16 18:40:16.329 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:40:16.731 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:40:18.330 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1798, "optime" : { "t" : 1358379619000, "i" : 994 }, "optimeDate" : ISODate("2013-01-16T23:40:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1787, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1787, "optime" : { "t" : 1358379614000, "i" : 8701 }, "optimeDate" : ISODate("2013-01-16T23:40:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:17Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1797, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1784, "optime" : { "t" : 1358379618000, "i" : 8495 }, "optimeDate" : ISODate("2013-01-16T23:40:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1784, "optime" : { "t" : 1358379614000, "i" : 8701 }, "optimeDate" : ISODate("2013-01-16T23:40:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1784, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:18Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1796, "optime" : { "t" : 1358379619000, "i" : 934 }, "optimeDate" : ISODate("2013-01-16T23:40:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:40:19.794 [conn69] end connection 127.0.0.4:52042 (4 connections now open) m31000| Wed Jan 16 18:40:19.794 [initandlisten] connection accepted from 127.0.0.4:54230 #70 (5 connections now open) m31000| Wed Jan 16 18:40:20.330 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:40:21.731 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:40:22.330 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1802, "optime" : { "t" : 1358379623000, "i" : 3207 }, "optimeDate" : ISODate("2013-01-16T23:40:23Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1791, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1791, "optime" : { "t" : 1358379622000, "i" : 226 }, "optimeDate" : ISODate("2013-01-16T23:40:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1801, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1788, "optime" : { "t" : 1358379622000, "i" : 6553 }, "optimeDate" : ISODate("2013-01-16T23:40:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1788, "optime" : { "t" : 1358379619000, "i" : 994 }, "optimeDate" : ISODate("2013-01-16T23:40:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1788, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:22Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1800, "optime" : { "t" : 1358379623000, "i" : 3207 }, "optimeDate" : ISODate("2013-01-16T23:40:23Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:40:24.331 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:40:26.331 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:40:26.732 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1806, "optime" : { "t" : 1358379627000, "i" : 4598 }, "optimeDate" : ISODate("2013-01-16T23:40:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1795, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1795, "optime" : { "t" : 1358379627000, "i" : 2149 }, "optimeDate" : ISODate("2013-01-16T23:40:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1805, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1792, "optime" : { "t" : 1358379626000, "i" : 5073 }, "optimeDate" : ISODate("2013-01-16T23:40:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1792, "optime" : { "t" : 1358379623000, "i" : 3207 }, "optimeDate" : ISODate("2013-01-16T23:40:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:27Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1792, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:26Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1804, "optime" : { "t" : 1358379627000, "i" : 4311 }, "optimeDate" : ISODate("2013-01-16T23:40:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:40:27.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:40:28.331 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:40:30.332 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 m31002| Wed Jan 16 18:40:31.733 [rsMgr] replSet info two primaries (transiently) { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1810, "optime" : { "t" : 1358379631000, "i" : 6107 }, "optimeDate" : ISODate("2013-01-16T23:40:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1799, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1799, "optime" : { "t" : 1358379627000, "i" : 4598 }, "optimeDate" : ISODate("2013-01-16T23:40:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:29Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1809, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1796, "optime" : { "t" : 1358379630000, "i" : 3387 }, "optimeDate" : ISODate("2013-01-16T23:40:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:31Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1796, "optime" : { "t" : 1358379627000, "i" : 4598 }, "optimeDate" : ISODate("2013-01-16T23:40:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1796, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1808, "optime" : { "t" : 1358379631000, "i" : 6107 }, "optimeDate" : ISODate("2013-01-16T23:40:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:40:32.332 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:40:32.944 [conn109] end connection 127.0.0.3:45290 (3 connections now open) m31002| Wed Jan 16 18:40:32.944 [initandlisten] connection accepted from 127.0.0.3:46743 #110 (4 connections now open) m31000| Wed Jan 16 18:40:34.333 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1814, "optime" : { "t" : 1358379635000, "i" : 8354 }, "optimeDate" : ISODate("2013-01-16T23:40:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1803, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1803, "optime" : { "t" : 1358379631000, "i" : 6107 }, "optimeDate" : ISODate("2013-01-16T23:40:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:35Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1813, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1800, "optime" : { "t" : 1358379634000, "i" : 1541 }, "optimeDate" : ISODate("2013-01-16T23:40:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1800, "optime" : { "t" : 1358379635000, "i" : 7518 }, "optimeDate" : ISODate("2013-01-16T23:40:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1800, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:34Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1812, "optime" : { "t" : 1358379635000, "i" : 7989 }, "optimeDate" : ISODate("2013-01-16T23:40:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:40:36.333 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:40:36.733 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:40:37.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:40:38.333 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:40:39.347 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:243844 121ms m31002| Wed Jan 16 18:40:39.742 [conn108] end connection 127.0.0.2:45193 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1819, "optime" : { "t" : 1358379640000, "i" : 819 }, "optimeDate" : ISODate("2013-01-16T23:40:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1808, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1808, "optime" : { "t" : 1358379635000, "i" : 8354 }, "optimeDate" : ISODate("2013-01-16T23:40:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:39Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1818, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1805, "optime" : { "t" : 1358379638000, "i" : 21 }, "optimeDate" : ISODate("2013-01-16T23:40:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:39Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1805, "optime" : { "t" : 1358379639000, "i" : 6673 }, "optimeDate" : ISODate("2013-01-16T23:40:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1805, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1817, "optime" : { "t" : 1358379640000, "i" : 819 }, "optimeDate" : ISODate("2013-01-16T23:40:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:40:40.243 [initandlisten] connection accepted from 127.0.0.2:34789 #111 (4 connections now open) m31000| Wed Jan 16 18:40:40.334 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:40:41.734 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:40:42.334 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1823, "optime" : { "t" : 1358379644000, "i" : 1240 }, "optimeDate" : ISODate("2013-01-16T23:40:44Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1812, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1812, "optime" : { "t" : 1358379640000, "i" : 819 }, "optimeDate" : ISODate("2013-01-16T23:40:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:43Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1822, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1809, "optime" : { "t" : 1358379640000, "i" : 819 }, "optimeDate" : ISODate("2013-01-16T23:40:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1809, "optime" : { "t" : 1358379643000, "i" : 6601 }, "optimeDate" : ISODate("2013-01-16T23:40:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1809, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:42Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1821, "optime" : { "t" : 1358379644000, "i" : 624 }, "optimeDate" : ISODate("2013-01-16T23:40:44Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:40:44.234 [conn107] end connection 127.0.0.4:39278 (2 connections now open) m31000| Wed Jan 16 18:40:44.335 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:40:44.734 [initandlisten] connection accepted from 127.0.0.4:50593 #109 (3 connections now open) m31001| Wed Jan 16 18:40:46.335 [conn108] end connection 127.0.0.2:49368 (2 connections now open) m31001| Wed Jan 16 18:40:46.335 [initandlisten] connection accepted from 127.0.0.2:52590 #110 (3 connections now open) m31000| Wed Jan 16 18:40:46.335 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:40:47.235 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:40:48.336 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1827, "optime" : { "t" : 1358379648000, "i" : 2959 }, "optimeDate" : ISODate("2013-01-16T23:40:48Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1816, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1816, "optime" : { "t" : 1358379647000, "i" : 5015 }, "optimeDate" : ISODate("2013-01-16T23:40:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:47Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1826, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1813, "optime" : { "t" : 1358379644000, "i" : 1240 }, "optimeDate" : ISODate("2013-01-16T23:40:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:47Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1813, "optime" : { "t" : 1358379647000, "i" : 5565 }, "optimeDate" : ISODate("2013-01-16T23:40:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1813, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1825, "optime" : { "t" : 1358379648000, "i" : 2959 }, "optimeDate" : ISODate("2013-01-16T23:40:48Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:40:49.799 [conn70] end connection 127.0.0.4:54230 (4 connections now open) m31000| Wed Jan 16 18:40:49.799 [initandlisten] connection accepted from 127.0.0.4:43772 #71 (5 connections now open) m31000| Wed Jan 16 18:40:50.336 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:40:51.621 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:224831 112ms m31000| Wed Jan 16 18:40:51.873 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:296228 148ms m31002| Wed Jan 16 18:40:52.236 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:40:52.336 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1831, "optime" : { "t" : 1358379652000, "i" : 3713 }, "optimeDate" : ISODate("2013-01-16T23:40:52Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1820, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1820, "optime" : { "t" : 1358379648000, "i" : 2959 }, "optimeDate" : ISODate("2013-01-16T23:40:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:51Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1830, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1817, "optime" : { "t" : 1358379648000, "i" : 2959 }, "optimeDate" : ISODate("2013-01-16T23:40:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:52Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1817, "optime" : { "t" : 1358379651000, "i" : 3776 }, "optimeDate" : ISODate("2013-01-16T23:40:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1817, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1829, "optime" : { "t" : 1358379652000, "i" : 3713 }, "optimeDate" : ISODate("2013-01-16T23:40:52Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:40:54.337 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:40:56.337 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1835, "optime" : { "t" : 1358379656000, "i" : 5476 }, "optimeDate" : ISODate("2013-01-16T23:40:56Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1824, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1824, "optime" : { "t" : 1358379652000, "i" : 3713 }, "optimeDate" : ISODate("2013-01-16T23:40:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:55Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1834, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1821, "optime" : { "t" : 1358379652000, "i" : 3713 }, "optimeDate" : ISODate("2013-01-16T23:40:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:40:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1821, "optime" : { "t" : 1358379655000, "i" : 2946 }, "optimeDate" : ISODate("2013-01-16T23:40:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1821, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:54Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1833, "optime" : { "t" : 1358379656000, "i" : 5418 }, "optimeDate" : ISODate("2013-01-16T23:40:56Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:40:57.236 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:40:58.337 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:40:59.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:41:00.338 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1839, "optime" : { "t" : 1358379660000, "i" : 6995 }, "optimeDate" : ISODate("2013-01-16T23:41:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1828, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1828, "optime" : { "t" : 1358379660000, "i" : 2035 }, "optimeDate" : ISODate("2013-01-16T23:41:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1838, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1825, "optime" : { "t" : 1358379656000, "i" : 5476 }, "optimeDate" : ISODate("2013-01-16T23:40:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:40:59Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1825, "optime" : { "t" : 1358379659000, "i" : 1336 }, "optimeDate" : ISODate("2013-01-16T23:40:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1825, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1837, "optime" : { "t" : 1358379660000, "i" : 6845 }, "optimeDate" : ISODate("2013-01-16T23:41:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:41:02.237 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:41:02.338 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:41:02.950 [conn110] end connection 127.0.0.3:46743 (3 connections now open) m31002| Wed Jan 16 18:41:02.950 [initandlisten] connection accepted from 127.0.0.3:55722 #112 (4 connections now open) m31000| Wed Jan 16 18:41:04.338 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1844, "optime" : { "t" : 1358379665000, "i" : 385 }, "optimeDate" : ISODate("2013-01-16T23:41:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1833, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1833, "optime" : { "t" : 1358379660000, "i" : 6995 }, "optimeDate" : ISODate("2013-01-16T23:41:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:03Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1843, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1830, "optime" : { "t" : 1358379664000, "i" : 7280 }, "optimeDate" : ISODate("2013-01-16T23:41:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1830, "optime" : { "t" : 1358379660000, "i" : 6995 }, "optimeDate" : ISODate("2013-01-16T23:41:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1830, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:04Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1842, "optime" : { "t" : 1358379665000, "i" : 385 }, "optimeDate" : ISODate("2013-01-16T23:41:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:41:06.339 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:41:07.237 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:41:08.339 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:41:09.360 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:377675 188ms { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1848, "optime" : { "t" : 1358379669000, "i" : 1752 }, "optimeDate" : ISODate("2013-01-16T23:41:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1837, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1837, "optime" : { "t" : 1358379665000, "i" : 385 }, "optimeDate" : ISODate("2013-01-16T23:41:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:07Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1847, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1834, "optime" : { "t" : 1358379668000, "i" : 7768 }, "optimeDate" : ISODate("2013-01-16T23:41:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1834, "optime" : { "t" : 1358379665000, "i" : 385 }, "optimeDate" : ISODate("2013-01-16T23:41:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1834, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:08Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1846, "optime" : { "t" : 1358379669000, "i" : 1752 }, "optimeDate" : ISODate("2013-01-16T23:41:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:41:09.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:41:10.339 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:41:12.238 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:41:12.340 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1852, "optime" : { "t" : 1358379673000, "i" : 4457 }, "optimeDate" : ISODate("2013-01-16T23:41:13Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1841, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1841, "optime" : { "t" : 1358379672000, "i" : 2918 }, "optimeDate" : ISODate("2013-01-16T23:41:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1851, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1838, "optime" : { "t" : 1358379672000, "i" : 4458 }, "optimeDate" : ISODate("2013-01-16T23:41:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1838, "optime" : { "t" : 1358379669000, "i" : 1752 }, "optimeDate" : ISODate("2013-01-16T23:41:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1838, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:12Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1850, "optime" : { "t" : 1358379673000, "i" : 4457 }, "optimeDate" : ISODate("2013-01-16T23:41:13Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:41:14.340 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:41:16.340 [conn110] end connection 127.0.0.2:52590 (2 connections now open) m31001| Wed Jan 16 18:41:16.341 [initandlisten] connection accepted from 127.0.0.2:42954 #111 (3 connections now open) m31000| Wed Jan 16 18:41:16.341 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:41:17.239 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1856, "optime" : { "t" : 1358379677000, "i" : 5574 }, "optimeDate" : ISODate("2013-01-16T23:41:17Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1845, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1845, "optime" : { "t" : 1358379673000, "i" : 4457 }, "optimeDate" : ISODate("2013-01-16T23:41:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:15Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:40:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1855, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1842, "optime" : { "t" : 1358379676000, "i" : 3171 }, "optimeDate" : ISODate("2013-01-16T23:41:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:17Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1842, "optime" : { "t" : 1358379673000, "i" : 4457 }, "optimeDate" : ISODate("2013-01-16T23:41:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1842, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1854, "optime" : { "t" : 1358379677000, "i" : 5574 }, "optimeDate" : ISODate("2013-01-16T23:41:17Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:41:17.748 [conn111] end connection 127.0.0.2:34789 (3 connections now open) m31002| Wed Jan 16 18:41:18.248 [initandlisten] connection accepted from 127.0.0.2:54866 #113 (4 connections now open) m31000| Wed Jan 16 18:41:18.341 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:41:19.805 [conn71] end connection 127.0.0.4:43772 (4 connections now open) m31000| Wed Jan 16 18:41:19.805 [initandlisten] connection accepted from 127.0.0.4:54127 #72 (5 connections now open) m31000| Wed Jan 16 18:41:20.342 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1860, "optime" : { "t" : 1358379681000, "i" : 7063 }, "optimeDate" : ISODate("2013-01-16T23:41:21Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1849, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1849, "optime" : { "t" : 1358379680000, "i" : 183 }, "optimeDate" : ISODate("2013-01-16T23:41:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1859, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1846, "optime" : { "t" : 1358379680000, "i" : 2746 }, "optimeDate" : ISODate("2013-01-16T23:41:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1846, "optime" : { "t" : 1358379677000, "i" : 5574 }, "optimeDate" : ISODate("2013-01-16T23:41:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1846, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:20Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1858, "optime" : { "t" : 1358379681000, "i" : 6417 }, "optimeDate" : ISODate("2013-01-16T23:41:21Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:41:22.239 [conn109] end connection 127.0.0.4:50593 (2 connections now open) m31000| Wed Jan 16 18:41:22.342 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:41:22.740 [initandlisten] connection accepted from 127.0.0.4:42914 #112 (3 connections now open) m31002| Wed Jan 16 18:41:22.740 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:41:24.342 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1864, "optime" : { "t" : 1358379685000, "i" : 8443 }, "optimeDate" : ISODate("2013-01-16T23:41:25Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1853, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1853, "optime" : { "t" : 1358379685000, "i" : 7012 }, "optimeDate" : ISODate("2013-01-16T23:41:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:25Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1863, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1850, "optime" : { "t" : 1358379684000, "i" : 1366 }, "optimeDate" : ISODate("2013-01-16T23:41:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:25Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:25Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1850, "optime" : { "t" : 1358379685000, "i" : 7075 }, "optimeDate" : ISODate("2013-01-16T23:41:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1850, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1862, "optime" : { "t" : 1358379685000, "i" : 8443 }, "optimeDate" : ISODate("2013-01-16T23:41:25Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:41:26.343 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:41:27.741 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:41:28.343 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:41:29.327 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:376892 188ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1868, "optime" : { "t" : 1358379689000, "i" : 9447 }, "optimeDate" : ISODate("2013-01-16T23:41:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1857, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1857, "optime" : { "t" : 1358379685000, "i" : 8443 }, "optimeDate" : ISODate("2013-01-16T23:41:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:29Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1867, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1854, "optime" : { "t" : 1358379688000, "i" : 430 }, "optimeDate" : ISODate("2013-01-16T23:41:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1854, "optime" : { "t" : 1358379689000, "i" : 7929 }, "optimeDate" : ISODate("2013-01-16T23:41:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1854, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:28Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1866, "optime" : { "t" : 1358379689000, "i" : 9447 }, "optimeDate" : ISODate("2013-01-16T23:41:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:41:30.344 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:41:31.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:41:32.344 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:41:32.741 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:41:32.955 [conn112] end connection 127.0.0.3:55722 (3 connections now open) m31002| Wed Jan 16 18:41:32.955 [initandlisten] connection accepted from 127.0.0.3:46957 #114 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1873, "optime" : { "t" : 1358379694000, "i" : 891 }, "optimeDate" : ISODate("2013-01-16T23:41:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1862, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1862, "optime" : { "t" : 1358379693000, "i" : 2301 }, "optimeDate" : ISODate("2013-01-16T23:41:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:33Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1872, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1859, "optime" : { "t" : 1358379689000, "i" : 9447 }, "optimeDate" : ISODate("2013-01-16T23:41:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1859, "optime" : { "t" : 1358379693000, "i" : 7405 }, "optimeDate" : ISODate("2013-01-16T23:41:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1859, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:32Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1871, "optime" : { "t" : 1358379694000, "i" : 218 }, "optimeDate" : ISODate("2013-01-16T23:41:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:41:34.344 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:41:36.345 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:41:37.742 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1877, "optime" : { "t" : 1358379698000, "i" : 2564 }, "optimeDate" : ISODate("2013-01-16T23:41:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1866, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1866, "optime" : { "t" : 1358379698000, "i" : 2357 }, "optimeDate" : ISODate("2013-01-16T23:41:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1876, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1863, "optime" : { "t" : 1358379694000, "i" : 891 }, "optimeDate" : ISODate("2013-01-16T23:41:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:37Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1863, "optime" : { "t" : 1358379697000, "i" : 6061 }, "optimeDate" : ISODate("2013-01-16T23:41:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1863, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1875, "optime" : { "t" : 1358379698000, "i" : 2564 }, "optimeDate" : ISODate("2013-01-16T23:41:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:41:38.345 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:41:40.346 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:41:41.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:41:42.346 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1881, "optime" : { "t" : 1358379702000, "i" : 3808 }, "optimeDate" : ISODate("2013-01-16T23:41:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1870, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1870, "optime" : { "t" : 1358379698000, "i" : 2564 }, "optimeDate" : ISODate("2013-01-16T23:41:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:41Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1880, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1867, "optime" : { "t" : 1358379698000, "i" : 2564 }, "optimeDate" : ISODate("2013-01-16T23:41:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1867, "optime" : { "t" : 1358379701000, "i" : 4279 }, "optimeDate" : ISODate("2013-01-16T23:41:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1867, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:40Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1879, "optime" : { "t" : 1358379702000, "i" : 3808 }, "optimeDate" : ISODate("2013-01-16T23:41:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:41:42.742 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:41:44.346 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:41:46.346 [conn111] end connection 127.0.0.2:42954 (2 connections now open) m31001| Wed Jan 16 18:41:46.347 [initandlisten] connection accepted from 127.0.0.2:56521 #113 (3 connections now open) m31000| Wed Jan 16 18:41:46.347 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1885, "optime" : { "t" : 1358379706000, "i" : 6081 }, "optimeDate" : ISODate("2013-01-16T23:41:46Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1874, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1874, "optime" : { "t" : 1358379705000, "i" : 1656 }, "optimeDate" : ISODate("2013-01-16T23:41:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:45Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1884, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1871, "optime" : { "t" : 1358379702000, "i" : 3808 }, "optimeDate" : ISODate("2013-01-16T23:41:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:45Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1871, "optime" : { "t" : 1358379705000, "i" : 1968 }, "optimeDate" : ISODate("2013-01-16T23:41:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1871, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1883, "optime" : { "t" : 1358379706000, "i" : 6081 }, "optimeDate" : ISODate("2013-01-16T23:41:46Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:41:47.743 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:41:48.347 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:41:49.809 [conn72] end connection 127.0.0.4:54127 (4 connections now open) m31000| Wed Jan 16 18:41:49.810 [initandlisten] connection accepted from 127.0.0.4:39123 #73 (5 connections now open) m31000| Wed Jan 16 18:41:50.347 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1889, "optime" : { "t" : 1358379710000, "i" : 7367 }, "optimeDate" : ISODate("2013-01-16T23:41:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1878, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1878, "optime" : { "t" : 1358379710000, "i" : 6336 }, "optimeDate" : ISODate("2013-01-16T23:41:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1888, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1875, "optime" : { "t" : 1358379706000, "i" : 6081 }, "optimeDate" : ISODate("2013-01-16T23:41:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:50Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1875, "optime" : { "t" : 1358379709000, "i" : 945 }, "optimeDate" : ISODate("2013-01-16T23:41:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1875, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1887, "optime" : { "t" : 1358379710000, "i" : 7367 }, "optimeDate" : ISODate("2013-01-16T23:41:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:41:52.348 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:41:52.744 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:41:54.348 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1894, "optime" : { "t" : 1358379715000, "i" : 454 }, "optimeDate" : ISODate("2013-01-16T23:41:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1883, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1883, "optime" : { "t" : 1358379710000, "i" : 7367 }, "optimeDate" : ISODate("2013-01-16T23:41:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:53Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1893, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1880, "optime" : { "t" : 1358379714000, "i" : 8682 }, "optimeDate" : ISODate("2013-01-16T23:41:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1880, "optime" : { "t" : 1358379710000, "i" : 7367 }, "optimeDate" : ISODate("2013-01-16T23:41:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1880, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:54Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1892, "optime" : { "t" : 1358379715000, "i" : 337 }, "optimeDate" : ISODate("2013-01-16T23:41:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:41:55.753 [conn113] end connection 127.0.0.2:54866 (3 connections now open) m31002| Wed Jan 16 18:41:56.253 [initandlisten] connection accepted from 127.0.0.2:37002 #115 (4 connections now open) m31000| Wed Jan 16 18:41:56.348 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:41:57.745 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:41:58.349 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1898, "optime" : { "t" : 1358379719000, "i" : 1050 }, "optimeDate" : ISODate("2013-01-16T23:41:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1887, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1887, "optime" : { "t" : 1358379718000, "i" : 6303 }, "optimeDate" : ISODate("2013-01-16T23:41:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1897, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1884, "optime" : { "t" : 1358379718000, "i" : 7896 }, "optimeDate" : ISODate("2013-01-16T23:41:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:41:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1884, "optime" : { "t" : 1358379715000, "i" : 454 }, "optimeDate" : ISODate("2013-01-16T23:41:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1884, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:41:58Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1896, "optime" : { "t" : 1358379719000, "i" : 296 }, "optimeDate" : ISODate("2013-01-16T23:41:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:42:00.245 [conn112] end connection 127.0.0.4:42914 (2 connections now open) m31000| Wed Jan 16 18:42:00.349 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:42:00.745 [initandlisten] connection accepted from 127.0.0.4:56150 #114 (3 connections now open) m31000| Wed Jan 16 18:42:02.349 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:42:02.960 [conn114] end connection 127.0.0.3:46957 (3 connections now open) m31002| Wed Jan 16 18:42:02.960 [initandlisten] connection accepted from 127.0.0.3:53895 #116 (4 connections now open) m31002| Wed Jan 16 18:42:03.245 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1902, "optime" : { "t" : 1358379723000, "i" : 2673 }, "optimeDate" : ISODate("2013-01-16T23:42:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1891, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1891, "optime" : { "t" : 1358379719000, "i" : 1050 }, "optimeDate" : ISODate("2013-01-16T23:41:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:01Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1901, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1888, "optime" : { "t" : 1358379722000, "i" : 6559 }, "optimeDate" : ISODate("2013-01-16T23:42:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:03Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1888, "optime" : { "t" : 1358379719000, "i" : 1050 }, "optimeDate" : ISODate("2013-01-16T23:41:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1888, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1900, "optime" : { "t" : 1358379723000, "i" : 2673 }, "optimeDate" : ISODate("2013-01-16T23:42:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:42:03.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:42:04.350 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:42:06.350 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1906, "optime" : { "t" : 1358379727000, "i" : 4704 }, "optimeDate" : ISODate("2013-01-16T23:42:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1895, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1895, "optime" : { "t" : 1358379723000, "i" : 2673 }, "optimeDate" : ISODate("2013-01-16T23:42:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1905, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1892, "optime" : { "t" : 1358379726000, "i" : 5187 }, "optimeDate" : ISODate("2013-01-16T23:42:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1892, "optime" : { "t" : 1358379723000, "i" : 2673 }, "optimeDate" : ISODate("2013-01-16T23:42:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1892, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:06Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1904, "optime" : { "t" : 1358379727000, "i" : 4339 }, "optimeDate" : ISODate("2013-01-16T23:42:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:42:08.246 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:42:08.351 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:42:10.351 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1910, "optime" : { "t" : 1358379731000, "i" : 5921 }, "optimeDate" : ISODate("2013-01-16T23:42:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1899, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1899, "optime" : { "t" : 1358379731000, "i" : 1997 }, "optimeDate" : ISODate("2013-01-16T23:42:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1909, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1896, "optime" : { "t" : 1358379730000, "i" : 3631 }, "optimeDate" : ISODate("2013-01-16T23:42:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1896, "optime" : { "t" : 1358379727000, "i" : 4704 }, "optimeDate" : ISODate("2013-01-16T23:42:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:11Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1896, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:10Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1908, "optime" : { "t" : 1358379731000, "i" : 5921 }, "optimeDate" : ISODate("2013-01-16T23:42:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:42:12.351 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:42:13.247 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:42:13.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:42:14.352 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1914, "optime" : { "t" : 1358379735000, "i" : 7624 }, "optimeDate" : ISODate("2013-01-16T23:42:15Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1903, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1903, "optime" : { "t" : 1358379731000, "i" : 5921 }, "optimeDate" : ISODate("2013-01-16T23:42:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:15Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:41:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1913, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1900, "optime" : { "t" : 1358379734000, "i" : 1536 }, "optimeDate" : ISODate("2013-01-16T23:42:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:15Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1900, "optime" : { "t" : 1358379735000, "i" : 6939 }, "optimeDate" : ISODate("2013-01-16T23:42:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1900, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1912, "optime" : { "t" : 1358379735000, "i" : 6886 }, "optimeDate" : ISODate("2013-01-16T23:42:15Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:42:16.352 [conn113] end connection 127.0.0.2:56521 (2 connections now open) m31001| Wed Jan 16 18:42:16.352 [initandlisten] connection accepted from 127.0.0.2:52790 #115 (3 connections now open) m31000| Wed Jan 16 18:42:16.353 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:42:18.247 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:42:18.353 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:42:19.382 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:410351 205ms m31000| Wed Jan 16 18:42:19.441 [FileAllocator] allocating new datafile /data/db/testReplSet-0/foo.5, filling with zeroes... m31000| Wed Jan 16 18:42:19.815 [conn73] end connection 127.0.0.4:39123 (4 connections now open) m31000| Wed Jan 16 18:42:19.815 [initandlisten] connection accepted from 127.0.0.4:46635 #74 (5 connections now open) STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:42:20.353 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:42:22.354 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:42:23.248 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:42:23.462 [FileAllocator] done allocating datafile /data/db/testReplSet-0/foo.5, size: 511MB, took 4.021 secs m31000| Wed Jan 16 18:42:23.462 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:4022114 4022ms m31002| Wed Jan 16 18:42:23.463 [FileAllocator] allocating new datafile /data/db/testReplSet-2/foo.5, filling with zeroes... { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1922, "optime" : { "t" : 1358379743000, "i" : 5745 }, "optimeDate" : ISODate("2013-01-16T23:42:23Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1911, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1911, "optime" : { "t" : 1358379739000, "i" : 2818 }, "optimeDate" : ISODate("2013-01-16T23:42:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:21Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1921, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1908, "optime" : { "t" : 1358379739000, "i" : 2818 }, "optimeDate" : ISODate("2013-01-16T23:42:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:23Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1908, "optime" : { "t" : 1358379739000, "i" : 2818 }, "optimeDate" : ISODate("2013-01-16T23:42:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:23Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1908, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1920, "optime" : { "t" : 1358379739000, "i" : 2818 }, "optimeDate" : ISODate("2013-01-16T23:42:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:42:24.354 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:42:26.354 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:42:28.249 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:42:28.355 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:42:30.355 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:42:31.207 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:8827881 4413ms m31000| Wed Jan 16 18:42:31.207 [conn10] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358377825000|1 } } cursorid:131228461816724 ntoreturn:0 keyUpdates:0 locks(micros) r:51 nreturned:17 reslen:1669 4415ms STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:42:32.355 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:42:32.965 [conn116] end connection 127.0.0.3:53895 (3 connections now open) m31002| Wed Jan 16 18:42:32.965 [initandlisten] connection accepted from 127.0.0.3:50928 #117 (4 connections now open) m31002| Wed Jan 16 18:42:33.250 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:42:33.563 [FileAllocator] done allocating datafile /data/db/testReplSet-2/foo.5, size: 511MB, took 10.099 secs m31000| Wed Jan 16 18:42:33.599 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:4573610 2286ms m31000| Wed Jan 16 18:42:33.599 [conn10] getmore local.oplog.rs query: { ts: { $gte: Timestamp 1358377825000|1 } } cursorid:131228461816724 ntoreturn:0 keyUpdates:0 locks(micros) r:32 nreturned:1 reslen:117 2289ms m31002| Wed Jan 16 18:42:33.758 [conn115] end connection 127.0.0.2:37002 (3 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1932, "optime" : { "t" : 1358379753000, "i" : 6514 }, "optimeDate" : ISODate("2013-01-16T23:42:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1921, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1921, "optime" : { "t" : 1358379739000, "i" : 2818 }, "optimeDate" : ISODate("2013-01-16T23:42:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:33Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1931, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1918, "optime" : { "t" : 1358379739000, "i" : 2818 }, "optimeDate" : ISODate("2013-01-16T23:42:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:33Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1918, "optime" : { "t" : 1358379753000, "i" : 4277 }, "optimeDate" : ISODate("2013-01-16T23:42:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1918, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1930, "optime" : { "t" : 1358379753000, "i" : 772 }, "optimeDate" : ISODate("2013-01-16T23:42:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:42:34.259 [initandlisten] connection accepted from 127.0.0.2:33108 #118 (4 connections now open) m31000| Wed Jan 16 18:42:34.356 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:42:35.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:42:36.356 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1937, "optime" : { "t" : 1358379758000, "i" : 1056 }, "optimeDate" : ISODate("2013-01-16T23:42:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1926, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1926, "optime" : { "t" : 1358379753000, "i" : 6514 }, "optimeDate" : ISODate("2013-01-16T23:42:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:37Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1936, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1923, "optime" : { "t" : 1358379756000, "i" : 6 }, "optimeDate" : ISODate("2013-01-16T23:42:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1923, "optime" : { "t" : 1358379757000, "i" : 7327 }, "optimeDate" : ISODate("2013-01-16T23:42:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1923, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:36Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1935, "optime" : { "t" : 1358379758000, "i" : 1056 }, "optimeDate" : ISODate("2013-01-16T23:42:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:42:38.250 [conn114] end connection 127.0.0.4:56150 (2 connections now open) m31000| Wed Jan 16 18:42:38.357 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:42:38.750 [initandlisten] connection accepted from 127.0.0.4:37964 #116 (3 connections now open) m31002| Wed Jan 16 18:42:38.751 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:42:40.357 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1941, "optime" : { "t" : 1358379762000, "i" : 2796 }, "optimeDate" : ISODate("2013-01-16T23:42:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1930, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1930, "optime" : { "t" : 1358379761000, "i" : 5481 }, "optimeDate" : ISODate("2013-01-16T23:42:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:41Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1940, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1927, "optime" : { "t" : 1358379758000, "i" : 1056 }, "optimeDate" : ISODate("2013-01-16T23:42:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:41Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1927, "optime" : { "t" : 1358379761000, "i" : 5984 }, "optimeDate" : ISODate("2013-01-16T23:42:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1927, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1939, "optime" : { "t" : 1358379762000, "i" : 2796 }, "optimeDate" : ISODate("2013-01-16T23:42:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:42:42.357 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:42:43.751 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:42:44.358 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:42:45.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 18:42:46.358 [conn115] end connection 127.0.0.2:52790 (2 connections now open) m31001| Wed Jan 16 18:42:46.358 [initandlisten] connection accepted from 127.0.0.2:35705 #117 (3 connections now open) m31000| Wed Jan 16 18:42:46.358 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1945, "optime" : { "t" : 1358379766000, "i" : 4342 }, "optimeDate" : ISODate("2013-01-16T23:42:46Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1934, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1934, "optime" : { "t" : 1358379762000, "i" : 2796 }, "optimeDate" : ISODate("2013-01-16T23:42:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:45Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1944, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1931, "optime" : { "t" : 1358379762000, "i" : 2796 }, "optimeDate" : ISODate("2013-01-16T23:42:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:46Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1931, "optime" : { "t" : 1358379765000, "i" : 3764 }, "optimeDate" : ISODate("2013-01-16T23:42:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1931, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1943, "optime" : { "t" : 1358379766000, "i" : 4342 }, "optimeDate" : ISODate("2013-01-16T23:42:46Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:42:48.359 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:42:48.752 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:42:49.820 [conn74] end connection 127.0.0.4:46635 (4 connections now open) m31000| Wed Jan 16 18:42:49.820 [initandlisten] connection accepted from 127.0.0.4:50055 #75 (5 connections now open) m31000| Wed Jan 16 18:42:50.359 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1949, "optime" : { "t" : 1358379770000, "i" : 5708 }, "optimeDate" : ISODate("2013-01-16T23:42:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1938, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1938, "optime" : { "t" : 1358379766000, "i" : 4342 }, "optimeDate" : ISODate("2013-01-16T23:42:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:49Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1948, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1935, "optime" : { "t" : 1358379766000, "i" : 4342 }, "optimeDate" : ISODate("2013-01-16T23:42:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1935, "optime" : { "t" : 1358379769000, "i" : 2539 }, "optimeDate" : ISODate("2013-01-16T23:42:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1935, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:48Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1947, "optime" : { "t" : 1358379770000, "i" : 5708 }, "optimeDate" : ISODate("2013-01-16T23:42:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:42:52.359 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:42:53.753 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:42:54.360 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1953, "optime" : { "t" : 1358379774000, "i" : 7199 }, "optimeDate" : ISODate("2013-01-16T23:42:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1942, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1942, "optime" : { "t" : 1358379774000, "i" : 2146 }, "optimeDate" : ISODate("2013-01-16T23:42:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1952, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1939, "optime" : { "t" : 1358379770000, "i" : 5708 }, "optimeDate" : ISODate("2013-01-16T23:42:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:53Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1939, "optime" : { "t" : 1358379773000, "i" : 1280 }, "optimeDate" : ISODate("2013-01-16T23:42:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1939, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1951, "optime" : { "t" : 1358379774000, "i" : 7199 }, "optimeDate" : ISODate("2013-01-16T23:42:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:42:56.360 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:42:58.360 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:42:58.753 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1957, "optime" : { "t" : 1358379778000, "i" : 8989 }, "optimeDate" : ISODate("2013-01-16T23:42:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1946, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1946, "optime" : { "t" : 1358379774000, "i" : 7199 }, "optimeDate" : ISODate("2013-01-16T23:42:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:57Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1956, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1943, "optime" : { "t" : 1358379778000, "i" : 7886 }, "optimeDate" : ISODate("2013-01-16T23:42:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:42:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1943, "optime" : { "t" : 1358379774000, "i" : 7199 }, "optimeDate" : ISODate("2013-01-16T23:42:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1943, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:42:58Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1955, "optime" : { "t" : 1358379778000, "i" : 8989 }, "optimeDate" : ISODate("2013-01-16T23:42:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:43:00.361 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:43:02.361 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:43:02.971 [conn117] end connection 127.0.0.3:50928 (3 connections now open) m31002| Wed Jan 16 18:43:02.971 [initandlisten] connection accepted from 127.0.0.3:40232 #119 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1962, "optime" : { "t" : 1358379783000, "i" : 1208 }, "optimeDate" : ISODate("2013-01-16T23:43:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1951, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1951, "optime" : { "t" : 1358379778000, "i" : 8989 }, "optimeDate" : ISODate("2013-01-16T23:42:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:01Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1961, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1948, "optime" : { "t" : 1358379782000, "i" : 8562 }, "optimeDate" : ISODate("2013-01-16T23:43:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1948, "optime" : { "t" : 1358379778000, "i" : 8989 }, "optimeDate" : ISODate("2013-01-16T23:42:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1948, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:02Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1960, "optime" : { "t" : 1358379783000, "i" : 934 }, "optimeDate" : ISODate("2013-01-16T23:43:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:43:03.754 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:43:04.361 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:43:06.362 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1966, "optime" : { "t" : 1358379787000, "i" : 2300 }, "optimeDate" : ISODate("2013-01-16T23:43:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1955, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1955, "optime" : { "t" : 1358379786000, "i" : 5345 }, "optimeDate" : ISODate("2013-01-16T23:43:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1965, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1952, "optime" : { "t" : 1358379786000, "i" : 7124 }, "optimeDate" : ISODate("2013-01-16T23:43:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1952, "optime" : { "t" : 1358379783000, "i" : 1208 }, "optimeDate" : ISODate("2013-01-16T23:43:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1952, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:06Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1964, "optime" : { "t" : 1358379787000, "i" : 1804 }, "optimeDate" : ISODate("2013-01-16T23:43:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:43:07.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:43:08.362 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:43:08.755 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:43:10.362 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1970, "optime" : { "t" : 1358379791000, "i" : 3693 }, "optimeDate" : ISODate("2013-01-16T23:43:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1959, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1959, "optime" : { "t" : 1358379787000, "i" : 2300 }, "optimeDate" : ISODate("2013-01-16T23:43:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:09Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1969, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1956, "optime" : { "t" : 1358379790000, "i" : 5684 }, "optimeDate" : ISODate("2013-01-16T23:43:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:11Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1956, "optime" : { "t" : 1358379787000, "i" : 2300 }, "optimeDate" : ISODate("2013-01-16T23:43:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1956, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1968, "optime" : { "t" : 1358379791000, "i" : 3693 }, "optimeDate" : ISODate("2013-01-16T23:43:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:43:11.764 [conn118] end connection 127.0.0.2:33108 (3 connections now open) m31002| Wed Jan 16 18:43:12.264 [initandlisten] connection accepted from 127.0.0.2:36496 #120 (4 connections now open) m31000| Wed Jan 16 18:43:12.363 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:43:13.755 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:43:14.363 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1974, "optime" : { "t" : 1358379795000, "i" : 5221 }, "optimeDate" : ISODate("2013-01-16T23:43:15Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1963, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1963, "optime" : { "t" : 1358379794000, "i" : 3159 }, "optimeDate" : ISODate("2013-01-16T23:43:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1973, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1960, "optime" : { "t" : 1358379794000, "i" : 4697 }, "optimeDate" : ISODate("2013-01-16T23:43:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1960, "optime" : { "t" : 1358379791000, "i" : 3693 }, "optimeDate" : ISODate("2013-01-16T23:43:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1960, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:14Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1972, "optime" : { "t" : 1358379795000, "i" : 4553 }, "optimeDate" : ISODate("2013-01-16T23:43:15Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:43:16.255 [conn116] end connection 127.0.0.4:37964 (2 connections now open) m31001| Wed Jan 16 18:43:16.363 [conn117] end connection 127.0.0.2:35705 (1 connection now open) m31001| Wed Jan 16 18:43:16.364 [initandlisten] connection accepted from 127.0.0.2:42770 #118 (2 connections now open) m31000| Wed Jan 16 18:43:16.364 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:43:16.756 [initandlisten] connection accepted from 127.0.0.4:35880 #119 (3 connections now open) m31001| Wed Jan 16 18:43:17.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:43:18.364 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:43:19.256 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1978, "optime" : { "t" : 1358379799000, "i" : 5819 }, "optimeDate" : ISODate("2013-01-16T23:43:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1967, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1967, "optime" : { "t" : 1358379795000, "i" : 5221 }, "optimeDate" : ISODate("2013-01-16T23:43:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:17Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1977, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1964, "optime" : { "t" : 1358379798000, "i" : 3662 }, "optimeDate" : ISODate("2013-01-16T23:43:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:19Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1964, "optime" : { "t" : 1358379795000, "i" : 5221 }, "optimeDate" : ISODate("2013-01-16T23:43:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1964, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1976, "optime" : { "t" : 1358379799000, "i" : 5699 }, "optimeDate" : ISODate("2013-01-16T23:43:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:43:19.825 [conn75] end connection 127.0.0.4:50055 (4 connections now open) m31000| Wed Jan 16 18:43:19.825 [initandlisten] connection accepted from 127.0.0.4:42801 #76 (5 connections now open) m31000| Wed Jan 16 18:43:20.365 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:43:22.365 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1982, "optime" : { "t" : 1358379803000, "i" : 7219 }, "optimeDate" : ISODate("2013-01-16T23:43:23Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1971, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1971, "optime" : { "t" : 1358379799000, "i" : 5819 }, "optimeDate" : ISODate("2013-01-16T23:43:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:42:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1981, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1968, "optime" : { "t" : 1358379802000, "i" : 2376 }, "optimeDate" : ISODate("2013-01-16T23:43:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1968, "optime" : { "t" : 1358379799000, "i" : 5819 }, "optimeDate" : ISODate("2013-01-16T23:43:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1968, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:22Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1980, "optime" : { "t" : 1358379803000, "i" : 7219 }, "optimeDate" : ISODate("2013-01-16T23:43:23Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:43:24.257 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:43:24.366 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:43:26.366 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1986, "optime" : { "t" : 1358379807000, "i" : 8297 }, "optimeDate" : ISODate("2013-01-16T23:43:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1975, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1975, "optime" : { "t" : 1358379807000, "i" : 2525 }, "optimeDate" : ISODate("2013-01-16T23:43:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:27Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1985, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1972, "optime" : { "t" : 1358379806000, "i" : 1535 }, "optimeDate" : ISODate("2013-01-16T23:43:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1972, "optime" : { "t" : 1358379807000, "i" : 7174 }, "optimeDate" : ISODate("2013-01-16T23:43:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1972, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:26Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1984, "optime" : { "t" : 1358379807000, "i" : 8297 }, "optimeDate" : ISODate("2013-01-16T23:43:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:43:28.366 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:43:29.258 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:43:30.367 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1991, "optime" : { "t" : 1358379812000, "i" : 747 }, "optimeDate" : ISODate("2013-01-16T23:43:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1980, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1980, "optime" : { "t" : 1358379807000, "i" : 8297 }, "optimeDate" : ISODate("2013-01-16T23:43:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:31Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1990, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1977, "optime" : { "t" : 1358379810000, "i" : 282 }, "optimeDate" : ISODate("2013-01-16T23:43:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:31Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1977, "optime" : { "t" : 1358379811000, "i" : 7221 }, "optimeDate" : ISODate("2013-01-16T23:43:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1977, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1989, "optime" : { "t" : 1358379812000, "i" : 747 }, "optimeDate" : ISODate("2013-01-16T23:43:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:43:32.367 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:43:32.976 [conn119] end connection 127.0.0.3:40232 (3 connections now open) m31002| Wed Jan 16 18:43:32.976 [initandlisten] connection accepted from 127.0.0.3:47340 #121 (4 connections now open) m31002| Wed Jan 16 18:43:34.258 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:43:34.368 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1995, "optime" : { "t" : 1358379816000, "i" : 2366 }, "optimeDate" : ISODate("2013-01-16T23:43:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1984, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1984, "optime" : { "t" : 1358379812000, "i" : 747 }, "optimeDate" : ISODate("2013-01-16T23:43:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:35Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1994, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1981, "optime" : { "t" : 1358379812000, "i" : 747 }, "optimeDate" : ISODate("2013-01-16T23:43:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1981, "optime" : { "t" : 1358379815000, "i" : 5901 }, "optimeDate" : ISODate("2013-01-16T23:43:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1981, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:34Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1993, "optime" : { "t" : 1358379816000, "i" : 2129 }, "optimeDate" : ISODate("2013-01-16T23:43:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:43:36.368 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:43:38.368 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:43:39.259 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:43:39.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:43:40.369 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1999, "optime" : { "t" : 1358379820000, "i" : 4056 }, "optimeDate" : ISODate("2013-01-16T23:43:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1988, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1988, "optime" : { "t" : 1358379819000, "i" : 4239 }, "optimeDate" : ISODate("2013-01-16T23:43:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:39Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1998, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1985, "optime" : { "t" : 1358379816000, "i" : 2366 }, "optimeDate" : ISODate("2013-01-16T23:43:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:39Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1985, "optime" : { "t" : 1358379819000, "i" : 4537 }, "optimeDate" : ISODate("2013-01-16T23:43:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1985, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1997, "optime" : { "t" : 1358379820000, "i" : 3991 }, "optimeDate" : ISODate("2013-01-16T23:43:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:43:42.369 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:43:43.729 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:441418 220ms m31002| Wed Jan 16 18:43:44.260 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:43:44.369 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2003, "optime" : { "t" : 1358379824000, "i" : 4639 }, "optimeDate" : ISODate("2013-01-16T23:43:44Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1992, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1992, "optime" : { "t" : 1358379820000, "i" : 4056 }, "optimeDate" : ISODate("2013-01-16T23:43:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:43Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2002, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1989, "optime" : { "t" : 1358379820000, "i" : 4056 }, "optimeDate" : ISODate("2013-01-16T23:43:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:44Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1989, "optime" : { "t" : 1358379823000, "i" : 3524 }, "optimeDate" : ISODate("2013-01-16T23:43:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1989, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2001, "optime" : { "t" : 1358379824000, "i" : 4639 }, "optimeDate" : ISODate("2013-01-16T23:43:44Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:43:46.370 [conn118] end connection 127.0.0.2:42770 (2 connections now open) m31001| Wed Jan 16 18:43:46.370 [initandlisten] connection accepted from 127.0.0.2:37208 #120 (4 connections now open) m31000| Wed Jan 16 18:43:46.370 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:43:48.370 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2007, "optime" : { "t" : 1358379828000, "i" : 6139 }, "optimeDate" : ISODate("2013-01-16T23:43:48Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1996, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1996, "optime" : { "t" : 1358379824000, "i" : 4639 }, "optimeDate" : ISODate("2013-01-16T23:43:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:47Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2006, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1993, "optime" : { "t" : 1358379824000, "i" : 4639 }, "optimeDate" : ISODate("2013-01-16T23:43:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1993, "optime" : { "t" : 1358379827000, "i" : 2392 }, "optimeDate" : ISODate("2013-01-16T23:43:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1993, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:46Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2005, "optime" : { "t" : 1358379828000, "i" : 6129 }, "optimeDate" : ISODate("2013-01-16T23:43:48Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:43:49.260 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:43:49.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:43:49.769 [conn120] end connection 127.0.0.2:36496 (3 connections now open) m31000| Wed Jan 16 18:43:49.830 [conn76] end connection 127.0.0.4:42801 (4 connections now open) m31000| Wed Jan 16 18:43:49.830 [initandlisten] connection accepted from 127.0.0.4:51496 #77 (5 connections now open) m31002| Wed Jan 16 18:43:50.269 [initandlisten] connection accepted from 127.0.0.2:40220 #122 (4 connections now open) m31000| Wed Jan 16 18:43:50.371 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:43:52.371 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2011, "optime" : { "t" : 1358379832000, "i" : 7426 }, "optimeDate" : ISODate("2013-01-16T23:43:52Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2000, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2000, "optime" : { "t" : 1358379832000, "i" : 6387 }, "optimeDate" : ISODate("2013-01-16T23:43:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2010, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 1997, "optime" : { "t" : 1358379828000, "i" : 6139 }, "optimeDate" : ISODate("2013-01-16T23:43:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:51Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1997, "optime" : { "t" : 1358379831000, "i" : 1042 }, "optimeDate" : ISODate("2013-01-16T23:43:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 1997, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2009, "optime" : { "t" : 1358379832000, "i" : 7426 }, "optimeDate" : ISODate("2013-01-16T23:43:52Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:43:54.261 [conn119] end connection 127.0.0.4:35880 (2 connections now open) m31000| Wed Jan 16 18:43:54.371 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:43:54.761 [initandlisten] connection accepted from 127.0.0.4:46795 #121 (3 connections now open) m31002| Wed Jan 16 18:43:54.761 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:43:56.372 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2015, "optime" : { "t" : 1358379836000, "i" : 9196 }, "optimeDate" : ISODate("2013-01-16T23:43:56Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2004, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2004, "optime" : { "t" : 1358379832000, "i" : 7426 }, "optimeDate" : ISODate("2013-01-16T23:43:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:55Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2015, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2002, "optime" : { "t" : 1358379836000, "i" : 8608 }, "optimeDate" : ISODate("2013-01-16T23:43:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:43:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2002, "optime" : { "t" : 1358379832000, "i" : 7426 }, "optimeDate" : ISODate("2013-01-16T23:43:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2002, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:43:56Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2014, "optime" : { "t" : 1358379836000, "i" : 9196 }, "optimeDate" : ISODate("2013-01-16T23:43:56Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:43:58.372 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:43:59.762 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:44:00.372 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2020, "optime" : { "t" : 1358379841000, "i" : 1434 }, "optimeDate" : ISODate("2013-01-16T23:44:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2009, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2009, "optime" : { "t" : 1358379840000, "i" : 1942 }, "optimeDate" : ISODate("2013-01-16T23:44:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2019, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2006, "optime" : { "t" : 1358379840000, "i" : 8153 }, "optimeDate" : ISODate("2013-01-16T23:44:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2006, "optime" : { "t" : 1358379836000, "i" : 9196 }, "optimeDate" : ISODate("2013-01-16T23:43:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2006, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2018, "optime" : { "t" : 1358379841000, "i" : 1434 }, "optimeDate" : ISODate("2013-01-16T23:44:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:44:02.373 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:44:02.982 [conn121] end connection 127.0.0.3:47340 (3 connections now open) m31002| Wed Jan 16 18:44:02.982 [initandlisten] connection accepted from 127.0.0.3:42981 #123 (4 connections now open) m31000| Wed Jan 16 18:44:04.373 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:44:04.763 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2024, "optime" : { "t" : 1358379845000, "i" : 2759 }, "optimeDate" : ISODate("2013-01-16T23:44:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2013, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2013, "optime" : { "t" : 1358379845000, "i" : 2095 }, "optimeDate" : ISODate("2013-01-16T23:44:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2023, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2010, "optime" : { "t" : 1358379844000, "i" : 7083 }, "optimeDate" : ISODate("2013-01-16T23:44:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2010, "optime" : { "t" : 1358379841000, "i" : 1434 }, "optimeDate" : ISODate("2013-01-16T23:44:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:05Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2010, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:04Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2022, "optime" : { "t" : 1358379845000, "i" : 2759 }, "optimeDate" : ISODate("2013-01-16T23:44:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:44:06.373 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:44:08.374 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2028, "optime" : { "t" : 1358379849000, "i" : 4096 }, "optimeDate" : ISODate("2013-01-16T23:44:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2017, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2017, "optime" : { "t" : 1358379845000, "i" : 2759 }, "optimeDate" : ISODate("2013-01-16T23:44:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:07Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2027, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2014, "optime" : { "t" : 1358379848000, "i" : 5757 }, "optimeDate" : ISODate("2013-01-16T23:44:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2014, "optime" : { "t" : 1358379845000, "i" : 2759 }, "optimeDate" : ISODate("2013-01-16T23:44:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2014, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:08Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2026, "optime" : { "t" : 1358379849000, "i" : 4008 }, "optimeDate" : ISODate("2013-01-16T23:44:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:44:09.763 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:44:10.374 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:44:11.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:44:12.374 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2032, "optime" : { "t" : 1358379853000, "i" : 5854 }, "optimeDate" : ISODate("2013-01-16T23:44:13Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2021, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2021, "optime" : { "t" : 1358379852000, "i" : 1744 }, "optimeDate" : ISODate("2013-01-16T23:44:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2031, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2018, "optime" : { "t" : 1358379852000, "i" : 3981 }, "optimeDate" : ISODate("2013-01-16T23:44:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2018, "optime" : { "t" : 1358379849000, "i" : 4096 }, "optimeDate" : ISODate("2013-01-16T23:44:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2018, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:12Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2030, "optime" : { "t" : 1358379853000, "i" : 5854 }, "optimeDate" : ISODate("2013-01-16T23:44:13Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:44:14.375 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:44:14.764 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:44:16.375 [conn120] end connection 127.0.0.2:37208 (2 connections now open) m31001| Wed Jan 16 18:44:16.375 [initandlisten] connection accepted from 127.0.0.2:38310 #122 (3 connections now open) m31000| Wed Jan 16 18:44:16.376 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2036, "optime" : { "t" : 1358379857000, "i" : 8086 }, "optimeDate" : ISODate("2013-01-16T23:44:17Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2025, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2025, "optime" : { "t" : 1358379857000, "i" : 7287 }, "optimeDate" : ISODate("2013-01-16T23:44:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:17Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2035, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2022, "optime" : { "t" : 1358379856000, "i" : 1755 }, "optimeDate" : ISODate("2013-01-16T23:44:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:17Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2022, "optime" : { "t" : 1358379857000, "i" : 7765 }, "optimeDate" : ISODate("2013-01-16T23:44:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2022, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2034, "optime" : { "t" : 1358379857000, "i" : 8086 }, "optimeDate" : ISODate("2013-01-16T23:44:17Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:44:18.376 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:44:19.765 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:44:19.836 [conn77] end connection 127.0.0.4:51496 (4 connections now open) m31000| Wed Jan 16 18:44:19.836 [initandlisten] connection accepted from 127.0.0.4:44096 #78 (5 connections now open) m31000| Wed Jan 16 18:44:20.376 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:44:21.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2041, "optime" : { "t" : 1358379862000, "i" : 12 }, "optimeDate" : ISODate("2013-01-16T23:44:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2030, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2030, "optime" : { "t" : 1358379857000, "i" : 8086 }, "optimeDate" : ISODate("2013-01-16T23:44:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:21Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2040, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2027, "optime" : { "t" : 1358379860000, "i" : 788 }, "optimeDate" : ISODate("2013-01-16T23:44:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2027, "optime" : { "t" : 1358379861000, "i" : 7395 }, "optimeDate" : ISODate("2013-01-16T23:44:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2027, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:20Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2039, "optime" : { "t" : 1358379861000, "i" : 8859 }, "optimeDate" : ISODate("2013-01-16T23:44:21Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:44:22.377 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:44:24.377 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:44:24.766 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2045, "optime" : { "t" : 1358379866000, "i" : 1630 }, "optimeDate" : ISODate("2013-01-16T23:44:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2034, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2034, "optime" : { "t" : 1358379865000, "i" : 1941 }, "optimeDate" : ISODate("2013-01-16T23:44:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:25Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2044, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2031, "optime" : { "t" : 1358379862000, "i" : 12 }, "optimeDate" : ISODate("2013-01-16T23:44:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2031, "optime" : { "t" : 1358379865000, "i" : 7154 }, "optimeDate" : ISODate("2013-01-16T23:44:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2031, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:24Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2043, "optime" : { "t" : 1358379866000, "i" : 1288 }, "optimeDate" : ISODate("2013-01-16T23:44:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:44:26.378 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:44:27.774 [conn122] end connection 127.0.0.2:40220 (3 connections now open) m31002| Wed Jan 16 18:44:28.274 [initandlisten] connection accepted from 127.0.0.2:51041 #124 (4 connections now open) m31000| Wed Jan 16 18:44:28.378 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:44:29.766 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:44:30.378 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2049, "optime" : { "t" : 1358379870000, "i" : 2658 }, "optimeDate" : ISODate("2013-01-16T23:44:30Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2038, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2038, "optime" : { "t" : 1358379866000, "i" : 1630 }, "optimeDate" : ISODate("2013-01-16T23:44:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:29Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:43:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2048, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2035, "optime" : { "t" : 1358379866000, "i" : 1630 }, "optimeDate" : ISODate("2013-01-16T23:44:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:29Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2035, "optime" : { "t" : 1358379869000, "i" : 5873 }, "optimeDate" : ISODate("2013-01-16T23:44:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2035, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2047, "optime" : { "t" : 1358379870000, "i" : 2606 }, "optimeDate" : ISODate("2013-01-16T23:44:30Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:44:32.266 [conn121] end connection 127.0.0.4:46795 (2 connections now open) m31000| Wed Jan 16 18:44:32.379 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:44:32.766 [initandlisten] connection accepted from 127.0.0.4:41652 #123 (3 connections now open) m31002| Wed Jan 16 18:44:32.987 [conn123] end connection 127.0.0.3:42981 (3 connections now open) m31002| Wed Jan 16 18:44:32.988 [initandlisten] connection accepted from 127.0.0.3:40139 #125 (4 connections now open) m31000| Wed Jan 16 18:44:34.379 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2053, "optime" : { "t" : 1358379874000, "i" : 4558 }, "optimeDate" : ISODate("2013-01-16T23:44:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2042, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2042, "optime" : { "t" : 1358379870000, "i" : 2658 }, "optimeDate" : ISODate("2013-01-16T23:44:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:33Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2052, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2039, "optime" : { "t" : 1358379870000, "i" : 2658 }, "optimeDate" : ISODate("2013-01-16T23:44:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2039, "optime" : { "t" : 1358379873000, "i" : 3937 }, "optimeDate" : ISODate("2013-01-16T23:44:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2039, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:32Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2051, "optime" : { "t" : 1358379874000, "i" : 4558 }, "optimeDate" : ISODate("2013-01-16T23:44:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:44:35.267 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:44:36.379 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:44:38.380 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2057, "optime" : { "t" : 1358379878000, "i" : 5667 }, "optimeDate" : ISODate("2013-01-16T23:44:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2046, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2046, "optime" : { "t" : 1358379878000, "i" : 2138 }, "optimeDate" : ISODate("2013-01-16T23:44:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2056, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2043, "optime" : { "t" : 1358379874000, "i" : 4558 }, "optimeDate" : ISODate("2013-01-16T23:44:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:37Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2043, "optime" : { "t" : 1358379877000, "i" : 2752 }, "optimeDate" : ISODate("2013-01-16T23:44:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2043, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2055, "optime" : { "t" : 1358379878000, "i" : 5667 }, "optimeDate" : ISODate("2013-01-16T23:44:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:44:40.268 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:44:40.380 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:44:42.380 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2061, "optime" : { "t" : 1358379882000, "i" : 7234 }, "optimeDate" : ISODate("2013-01-16T23:44:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2050, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2050, "optime" : { "t" : 1358379878000, "i" : 5667 }, "optimeDate" : ISODate("2013-01-16T23:44:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:41Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2060, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2047, "optime" : { "t" : 1358379878000, "i" : 5667 }, "optimeDate" : ISODate("2013-01-16T23:44:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:42Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2047, "optime" : { "t" : 1358379881000, "i" : 1439 }, "optimeDate" : ISODate("2013-01-16T23:44:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2047, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2059, "optime" : { "t" : 1358379882000, "i" : 7007 }, "optimeDate" : ISODate("2013-01-16T23:44:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:44:43.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:44:44.381 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:44:45.269 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:44:46.381 [conn122] end connection 127.0.0.2:38310 (2 connections now open) m31001| Wed Jan 16 18:44:46.381 [initandlisten] connection accepted from 127.0.0.2:48849 #124 (3 connections now open) m31000| Wed Jan 16 18:44:46.381 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2066, "optime" : { "t" : 1358379887000, "i" : 328 }, "optimeDate" : ISODate("2013-01-16T23:44:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2055, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2055, "optime" : { "t" : 1358379882000, "i" : 7234 }, "optimeDate" : ISODate("2013-01-16T23:44:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:45Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2065, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2052, "optime" : { "t" : 1358379886000, "i" : 8340 }, "optimeDate" : ISODate("2013-01-16T23:44:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2052, "optime" : { "t" : 1358379882000, "i" : 7234 }, "optimeDate" : ISODate("2013-01-16T23:44:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2052, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:46Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2064, "optime" : { "t" : 1358379887000, "i" : 320 }, "optimeDate" : ISODate("2013-01-16T23:44:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:44:48.382 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:44:49.841 [conn78] end connection 127.0.0.4:44096 (4 connections now open) m31000| Wed Jan 16 18:44:49.841 [initandlisten] connection accepted from 127.0.0.4:36306 #79 (5 connections now open) m31002| Wed Jan 16 18:44:50.269 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:44:50.382 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2070, "optime" : { "t" : 1358379891000, "i" : 2279 }, "optimeDate" : ISODate("2013-01-16T23:44:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2059, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2059, "optime" : { "t" : 1358379890000, "i" : 5935 }, "optimeDate" : ISODate("2013-01-16T23:44:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2069, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2056, "optime" : { "t" : 1358379890000, "i" : 7717 }, "optimeDate" : ISODate("2013-01-16T23:44:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2056, "optime" : { "t" : 1358379887000, "i" : 328 }, "optimeDate" : ISODate("2013-01-16T23:44:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2056, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:50Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2068, "optime" : { "t" : 1358379891000, "i" : 2279 }, "optimeDate" : ISODate("2013-01-16T23:44:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:44:52.382 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:44:53.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:44:54.383 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:44:55.270 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2074, "optime" : { "t" : 1358379895000, "i" : 3778 }, "optimeDate" : ISODate("2013-01-16T23:44:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2063, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2063, "optime" : { "t" : 1358379891000, "i" : 2279 }, "optimeDate" : ISODate("2013-01-16T23:44:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:53Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2073, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2060, "optime" : { "t" : 1358379894000, "i" : 6222 }, "optimeDate" : ISODate("2013-01-16T23:44:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:55Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2060, "optime" : { "t" : 1358379891000, "i" : 2279 }, "optimeDate" : ISODate("2013-01-16T23:44:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2060, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2072, "optime" : { "t" : 1358379895000, "i" : 3778 }, "optimeDate" : ISODate("2013-01-16T23:44:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:44:56.383 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:44:58.384 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2078, "optime" : { "t" : 1358379899000, "i" : 4950 }, "optimeDate" : ISODate("2013-01-16T23:44:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2067, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2067, "optime" : { "t" : 1358379895000, "i" : 3778 }, "optimeDate" : ISODate("2013-01-16T23:44:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2077, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2064, "optime" : { "t" : 1358379898000, "i" : 4839 }, "optimeDate" : ISODate("2013-01-16T23:44:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:44:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2064, "optime" : { "t" : 1358379895000, "i" : 3778 }, "optimeDate" : ISODate("2013-01-16T23:44:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2064, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:44:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:44:58Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2076, "optime" : { "t" : 1358379899000, "i" : 4950 }, "optimeDate" : ISODate("2013-01-16T23:44:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:45:00.271 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:45:00.384 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:45:02.384 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:45:02.993 [conn125] end connection 127.0.0.3:40139 (3 connections now open) m31002| Wed Jan 16 18:45:02.993 [initandlisten] connection accepted from 127.0.0.3:44028 #126 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2082, "optime" : { "t" : 1358379903000, "i" : 6052 }, "optimeDate" : ISODate("2013-01-16T23:45:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2071, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2071, "optime" : { "t" : 1358379903000, "i" : 2413 }, "optimeDate" : ISODate("2013-01-16T23:45:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2081, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2068, "optime" : { "t" : 1358379902000, "i" : 3945 }, "optimeDate" : ISODate("2013-01-16T23:45:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2068, "optime" : { "t" : 1358379899000, "i" : 4950 }, "optimeDate" : ISODate("2013-01-16T23:44:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:03Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2068, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:02Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2080, "optime" : { "t" : 1358379903000, "i" : 5605 }, "optimeDate" : ISODate("2013-01-16T23:45:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:45:04.385 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:45:05.271 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:45:05.780 [conn124] end connection 127.0.0.2:51041 (3 connections now open) m31002| Wed Jan 16 18:45:06.280 [initandlisten] connection accepted from 127.0.0.2:35461 #127 (4 connections now open) m31000| Wed Jan 16 18:45:06.385 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2086, "optime" : { "t" : 1358379907000, "i" : 6873 }, "optimeDate" : ISODate("2013-01-16T23:45:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2075, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2075, "optime" : { "t" : 1358379903000, "i" : 6052 }, "optimeDate" : ISODate("2013-01-16T23:45:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2085, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2072, "optime" : { "t" : 1358379906000, "i" : 2554 }, "optimeDate" : ISODate("2013-01-16T23:45:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:07Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2072, "optime" : { "t" : 1358379903000, "i" : 6052 }, "optimeDate" : ISODate("2013-01-16T23:45:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2072, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2084, "optime" : { "t" : 1358379907000, "i" : 6716 }, "optimeDate" : ISODate("2013-01-16T23:45:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:45:08.385 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:45:10.272 [conn123] end connection 127.0.0.4:41652 (2 connections now open) m31000| Wed Jan 16 18:45:10.386 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:45:10.772 [initandlisten] connection accepted from 127.0.0.4:47189 #125 (3 connections now open) m31002| Wed Jan 16 18:45:10.772 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2090, "optime" : { "t" : 1358379911000, "i" : 8201 }, "optimeDate" : ISODate("2013-01-16T23:45:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2079, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2079, "optime" : { "t" : 1358379911000, "i" : 2125 }, "optimeDate" : ISODate("2013-01-16T23:45:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:11Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2089, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2076, "optime" : { "t" : 1358379910000, "i" : 1715 }, "optimeDate" : ISODate("2013-01-16T23:45:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2076, "optime" : { "t" : 1358379911000, "i" : 7063 }, "optimeDate" : ISODate("2013-01-16T23:45:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2076, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:10Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2088, "optime" : { "t" : 1358379911000, "i" : 8192 }, "optimeDate" : ISODate("2013-01-16T23:45:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:45:12.386 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:45:14.387 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:45:15.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:45:15.773 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2095, "optime" : { "t" : 1358379916000, "i" : 1122 }, "optimeDate" : ISODate("2013-01-16T23:45:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2084, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2084, "optime" : { "t" : 1358379911000, "i" : 8201 }, "optimeDate" : ISODate("2013-01-16T23:45:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:15Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2094, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2081, "optime" : { "t" : 1358379914000, "i" : 20 }, "optimeDate" : ISODate("2013-01-16T23:45:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:15Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2081, "optime" : { "t" : 1358379915000, "i" : 7531 }, "optimeDate" : ISODate("2013-01-16T23:45:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2081, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2093, "optime" : { "t" : 1358379916000, "i" : 1122 }, "optimeDate" : ISODate("2013-01-16T23:45:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:45:16.387 [conn124] end connection 127.0.0.2:48849 (2 connections now open) m31001| Wed Jan 16 18:45:16.387 [initandlisten] connection accepted from 127.0.0.2:38100 #126 (3 connections now open) m31000| Wed Jan 16 18:45:16.387 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:45:18.388 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:45:19.846 [conn79] end connection 127.0.0.4:36306 (4 connections now open) m31000| Wed Jan 16 18:45:19.847 [initandlisten] connection accepted from 127.0.0.4:46436 #80 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2099, "optime" : { "t" : 1358379920000, "i" : 2324 }, "optimeDate" : ISODate("2013-01-16T23:45:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2088, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2088, "optime" : { "t" : 1358379916000, "i" : 1122 }, "optimeDate" : ISODate("2013-01-16T23:45:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:19Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2098, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2085, "optime" : { "t" : 1358379916000, "i" : 1122 }, "optimeDate" : ISODate("2013-01-16T23:45:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2085, "optime" : { "t" : 1358379919000, "i" : 6351 }, "optimeDate" : ISODate("2013-01-16T23:45:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2085, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:18Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2097, "optime" : { "t" : 1358379920000, "i" : 2324 }, "optimeDate" : ISODate("2013-01-16T23:45:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:45:20.388 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:45:20.773 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:45:22.389 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:45:23.757 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:410511 205ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2103, "optime" : { "t" : 1358379924000, "i" : 2190 }, "optimeDate" : ISODate("2013-01-16T23:45:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2092, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2092, "optime" : { "t" : 1358379923000, "i" : 153 }, "optimeDate" : ISODate("2013-01-16T23:45:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:23Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2102, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2089, "optime" : { "t" : 1358379920000, "i" : 2324 }, "optimeDate" : ISODate("2013-01-16T23:45:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:23Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2089, "optime" : { "t" : 1358379923000, "i" : 5609 }, "optimeDate" : ISODate("2013-01-16T23:45:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2089, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2101, "optime" : { "t" : 1358379924000, "i" : 2190 }, "optimeDate" : ISODate("2013-01-16T23:45:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:45:24.389 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:45:25.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:45:25.774 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:45:26.389 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2107, "optime" : { "t" : 1358379928000, "i" : 3062 }, "optimeDate" : ISODate("2013-01-16T23:45:28Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2096, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2096, "optime" : { "t" : 1358379924000, "i" : 2190 }, "optimeDate" : ISODate("2013-01-16T23:45:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:27Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2106, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2093, "optime" : { "t" : 1358379924000, "i" : 2190 }, "optimeDate" : ISODate("2013-01-16T23:45:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:28Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:28Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2093, "optime" : { "t" : 1358379927000, "i" : 5527 }, "optimeDate" : ISODate("2013-01-16T23:45:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2093, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2105, "optime" : { "t" : 1358379928000, "i" : 3062 }, "optimeDate" : ISODate("2013-01-16T23:45:28Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:45:28.390 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:45:30.390 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:45:30.775 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:45:32.390 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2111, "optime" : { "t" : 1358379932000, "i" : 4116 }, "optimeDate" : ISODate("2013-01-16T23:45:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2100, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2100, "optime" : { "t" : 1358379928000, "i" : 3062 }, "optimeDate" : ISODate("2013-01-16T23:45:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:31Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2110, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2097, "optime" : { "t" : 1358379928000, "i" : 3062 }, "optimeDate" : ISODate("2013-01-16T23:45:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2097, "optime" : { "t" : 1358379931000, "i" : 4338 }, "optimeDate" : ISODate("2013-01-16T23:45:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2097, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:30Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2109, "optime" : { "t" : 1358379932000, "i" : 3935 }, "optimeDate" : ISODate("2013-01-16T23:45:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:45:32.999 [conn126] end connection 127.0.0.3:44028 (3 connections now open) m31002| Wed Jan 16 18:45:32.999 [initandlisten] connection accepted from 127.0.0.3:32830 #128 (4 connections now open) m31000| Wed Jan 16 18:45:34.391 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:45:35.776 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:45:36.391 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2115, "optime" : { "t" : 1358379936000, "i" : 5882 }, "optimeDate" : ISODate("2013-01-16T23:45:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2104, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2104, "optime" : { "t" : 1358379936000, "i" : 2438 }, "optimeDate" : ISODate("2013-01-16T23:45:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2114, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2101, "optime" : { "t" : 1358379932000, "i" : 4116 }, "optimeDate" : ISODate("2013-01-16T23:45:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:35Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2101, "optime" : { "t" : 1358379935000, "i" : 2885 }, "optimeDate" : ISODate("2013-01-16T23:45:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2101, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2113, "optime" : { "t" : 1358379936000, "i" : 5645 }, "optimeDate" : ISODate("2013-01-16T23:45:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:45:38.392 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:45:40.392 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:45:40.776 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2119, "optime" : { "t" : 1358379940000, "i" : 7221 }, "optimeDate" : ISODate("2013-01-16T23:45:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2108, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2108, "optime" : { "t" : 1358379936000, "i" : 5882 }, "optimeDate" : ISODate("2013-01-16T23:45:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:39Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2118, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2105, "optime" : { "t" : 1358379936000, "i" : 5882 }, "optimeDate" : ISODate("2013-01-16T23:45:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:40Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2105, "optime" : { "t" : 1358379939000, "i" : 1332 }, "optimeDate" : ISODate("2013-01-16T23:45:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2105, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2117, "optime" : { "t" : 1358379940000, "i" : 7221 }, "optimeDate" : ISODate("2013-01-16T23:45:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:45:42.392 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:45:43.785 [conn127] end connection 127.0.0.2:35461 (3 connections now open) m31002| Wed Jan 16 18:45:44.285 [initandlisten] connection accepted from 127.0.0.2:54475 #129 (4 connections now open) m31000| Wed Jan 16 18:45:44.393 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2123, "optime" : { "t" : 1358379944000, "i" : 8386 }, "optimeDate" : ISODate("2013-01-16T23:45:44Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2112, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2112, "optime" : { "t" : 1358379944000, "i" : 2070 }, "optimeDate" : ISODate("2013-01-16T23:45:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2122, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2109, "optime" : { "t" : 1358379940000, "i" : 7221 }, "optimeDate" : ISODate("2013-01-16T23:45:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:43Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2109, "optime" : { "t" : 1358379943000, "i" : 14 }, "optimeDate" : ISODate("2013-01-16T23:45:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2109, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2121, "optime" : { "t" : 1358379944000, "i" : 8386 }, "optimeDate" : ISODate("2013-01-16T23:45:44Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:45:45.777 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:45:46.393 [conn126] end connection 127.0.0.2:38100 (2 connections now open) m31001| Wed Jan 16 18:45:46.393 [initandlisten] connection accepted from 127.0.0.2:55236 #127 (3 connections now open) m31000| Wed Jan 16 18:45:46.393 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:45:47.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 18:45:48.277 [conn125] end connection 127.0.0.4:47189 (2 connections now open) m31000| Wed Jan 16 18:45:48.394 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:45:48.777 [initandlisten] connection accepted from 127.0.0.4:54980 #128 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2128, "optime" : { "t" : 1358379949000, "i" : 1133 }, "optimeDate" : ISODate("2013-01-16T23:45:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2117, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2117, "optime" : { "t" : 1358379944000, "i" : 8386 }, "optimeDate" : ISODate("2013-01-16T23:45:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:47Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2127, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2114, "optime" : { "t" : 1358379948000, "i" : 8253 }, "optimeDate" : ISODate("2013-01-16T23:45:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2114, "optime" : { "t" : 1358379944000, "i" : 8386 }, "optimeDate" : ISODate("2013-01-16T23:45:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2114, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:48Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2126, "optime" : { "t" : 1358379949000, "i" : 1133 }, "optimeDate" : ISODate("2013-01-16T23:45:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:45:49.851 [conn80] end connection 127.0.0.4:46436 (4 connections now open) m31000| Wed Jan 16 18:45:49.852 [initandlisten] connection accepted from 127.0.0.4:38642 #81 (5 connections now open) m31000| Wed Jan 16 18:45:50.394 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:45:51.278 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:45:52.394 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:45:53.203 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:247995 124ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2132, "optime" : { "t" : 1358379953000, "i" : 2516 }, "optimeDate" : ISODate("2013-01-16T23:45:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2121, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2121, "optime" : { "t" : 1358379949000, "i" : 1133 }, "optimeDate" : ISODate("2013-01-16T23:45:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:51Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2131, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2118, "optime" : { "t" : 1358379952000, "i" : 7436 }, "optimeDate" : ISODate("2013-01-16T23:45:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2118, "optime" : { "t" : 1358379949000, "i" : 1133 }, "optimeDate" : ISODate("2013-01-16T23:45:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2118, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:52Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2130, "optime" : { "t" : 1358379953000, "i" : 785 }, "optimeDate" : ISODate("2013-01-16T23:45:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:45:54.395 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:45:56.279 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:45:56.395 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2136, "optime" : { "t" : 1358379957000, "i" : 3759 }, "optimeDate" : ISODate("2013-01-16T23:45:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2125, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2125, "optime" : { "t" : 1358379956000, "i" : 4124 }, "optimeDate" : ISODate("2013-01-16T23:45:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2135, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2122, "optime" : { "t" : 1358379956000, "i" : 5812 }, "optimeDate" : ISODate("2013-01-16T23:45:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:45:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2122, "optime" : { "t" : 1358379953000, "i" : 2516 }, "optimeDate" : ISODate("2013-01-16T23:45:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2122, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2134, "optime" : { "t" : 1358379957000, "i" : 3681 }, "optimeDate" : ISODate("2013-01-16T23:45:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:45:57.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:45:58.395 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:46:00.396 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:46:01.279 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2140, "optime" : { "t" : 1358379961000, "i" : 5274 }, "optimeDate" : ISODate("2013-01-16T23:46:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2129, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2129, "optime" : { "t" : 1358379957000, "i" : 3759 }, "optimeDate" : ISODate("2013-01-16T23:45:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:45:59Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2139, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2126, "optime" : { "t" : 1358379960000, "i" : 4505 }, "optimeDate" : ISODate("2013-01-16T23:46:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:01Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2126, "optime" : { "t" : 1358379957000, "i" : 3759 }, "optimeDate" : ISODate("2013-01-16T23:45:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2126, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2138, "optime" : { "t" : 1358379961000, "i" : 5274 }, "optimeDate" : ISODate("2013-01-16T23:46:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:46:02.396 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:46:03.004 [conn128] end connection 127.0.0.3:32830 (3 connections now open) m31002| Wed Jan 16 18:46:03.004 [initandlisten] connection accepted from 127.0.0.3:58085 #130 (4 connections now open) m31000| Wed Jan 16 18:46:04.396 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2144, "optime" : { "t" : 1358379965000, "i" : 6146 }, "optimeDate" : ISODate("2013-01-16T23:46:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2133, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2133, "optime" : { "t" : 1358379961000, "i" : 5274 }, "optimeDate" : ISODate("2013-01-16T23:46:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:45:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2143, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2130, "optime" : { "t" : 1358379965000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:46:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2130, "optime" : { "t" : 1358379961000, "i" : 5274 }, "optimeDate" : ISODate("2013-01-16T23:46:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2130, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:05Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2142, "optime" : { "t" : 1358379965000, "i" : 5401 }, "optimeDate" : ISODate("2013-01-16T23:46:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:46:06.280 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:46:06.397 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:46:08.397 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2148, "optime" : { "t" : 1358379969000, "i" : 7226 }, "optimeDate" : ISODate("2013-01-16T23:46:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2137, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2137, "optime" : { "t" : 1358379969000, "i" : 1876 }, "optimeDate" : ISODate("2013-01-16T23:46:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2147, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2134, "optime" : { "t" : 1358379968000, "i" : 1973 }, "optimeDate" : ISODate("2013-01-16T23:46:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2134, "optime" : { "t" : 1358379965000, "i" : 6146 }, "optimeDate" : ISODate("2013-01-16T23:46:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:09Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2134, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:09Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2146, "optime" : { "t" : 1358379969000, "i" : 7226 }, "optimeDate" : ISODate("2013-01-16T23:46:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:46:10.397 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:46:11.281 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:46:12.398 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2152, "optime" : { "t" : 1358379973000, "i" : 8661 }, "optimeDate" : ISODate("2013-01-16T23:46:13Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2141, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2141, "optime" : { "t" : 1358379969000, "i" : 7226 }, "optimeDate" : ISODate("2013-01-16T23:46:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:13Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2151, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2138, "optime" : { "t" : 1358379973000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:46:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:13Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2138, "optime" : { "t" : 1358379973000, "i" : 7509 }, "optimeDate" : ISODate("2013-01-16T23:46:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2138, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2150, "optime" : { "t" : 1358379973000, "i" : 8661 }, "optimeDate" : ISODate("2013-01-16T23:46:13Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:46:14.398 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:46:16.281 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:46:16.399 [conn127] end connection 127.0.0.2:55236 (2 connections now open) m31001| Wed Jan 16 18:46:16.399 [initandlisten] connection accepted from 127.0.0.2:33034 #129 (3 connections now open) m31000| Wed Jan 16 18:46:16.399 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2157, "optime" : { "t" : 1358379978000, "i" : 284 }, "optimeDate" : ISODate("2013-01-16T23:46:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2146, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2146, "optime" : { "t" : 1358379973000, "i" : 8661 }, "optimeDate" : ISODate("2013-01-16T23:46:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:17Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2156, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2143, "optime" : { "t" : 1358379977000, "i" : 7 }, "optimeDate" : ISODate("2013-01-16T23:46:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2143, "optime" : { "t" : 1358379977000, "i" : 7552 }, "optimeDate" : ISODate("2013-01-16T23:46:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2143, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2155, "optime" : { "t" : 1358379977000, "i" : 9301 }, "optimeDate" : ISODate("2013-01-16T23:46:17Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:46:18.399 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:46:19.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:46:19.857 [conn81] end connection 127.0.0.4:38642 (4 connections now open) m31000| Wed Jan 16 18:46:19.857 [initandlisten] connection accepted from 127.0.0.4:50299 #82 (5 connections now open) m31000| Wed Jan 16 18:46:20.400 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:46:21.282 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:46:21.790 [conn129] end connection 127.0.0.2:54475 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2161, "optime" : { "t" : 1358379982000, "i" : 2264 }, "optimeDate" : ISODate("2013-01-16T23:46:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2150, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2150, "optime" : { "t" : 1358379978000, "i" : 284 }, "optimeDate" : ISODate("2013-01-16T23:46:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:21Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2160, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2147, "optime" : { "t" : 1358379978000, "i" : 284 }, "optimeDate" : ISODate("2013-01-16T23:46:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:21Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2147, "optime" : { "t" : 1358379981000, "i" : 6801 }, "optimeDate" : ISODate("2013-01-16T23:46:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2147, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2159, "optime" : { "t" : 1358379982000, "i" : 2264 }, "optimeDate" : ISODate("2013-01-16T23:46:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:46:22.290 [initandlisten] connection accepted from 127.0.0.2:36383 #131 (4 connections now open) m31000| Wed Jan 16 18:46:22.400 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:46:24.401 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:46:26.282 [conn128] end connection 127.0.0.4:54980 (2 connections now open) STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:46:26.401 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2165, "optime" : { "t" : 1358379986000, "i" : 3359 }, "optimeDate" : ISODate("2013-01-16T23:46:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2154, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2154, "optime" : { "t" : 1358379982000, "i" : 2264 }, "optimeDate" : ISODate("2013-01-16T23:46:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:25Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2164, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2151, "optime" : { "t" : 1358379982000, "i" : 2264 }, "optimeDate" : ISODate("2013-01-16T23:46:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2151, "optime" : { "t" : 1358379985000, "i" : 5328 }, "optimeDate" : ISODate("2013-01-16T23:46:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2151, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2163, "optime" : { "t" : 1358379986000, "i" : 3359 }, "optimeDate" : ISODate("2013-01-16T23:46:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:46:26.783 [initandlisten] connection accepted from 127.0.0.4:59086 #130 (3 connections now open) m31002| Wed Jan 16 18:46:26.783 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:46:28.401 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:46:29.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:46:30.402 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2169, "optime" : { "t" : 1358379990000, "i" : 5250 }, "optimeDate" : ISODate("2013-01-16T23:46:30Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2158, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2158, "optime" : { "t" : 1358379989000, "i" : 2961 }, "optimeDate" : ISODate("2013-01-16T23:46:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:29Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2168, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2155, "optime" : { "t" : 1358379986000, "i" : 3359 }, "optimeDate" : ISODate("2013-01-16T23:46:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:29Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2155, "optime" : { "t" : 1358379989000, "i" : 3925 }, "optimeDate" : ISODate("2013-01-16T23:46:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2155, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2167, "optime" : { "t" : 1358379990000, "i" : 5250 }, "optimeDate" : ISODate("2013-01-16T23:46:30Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:46:31.784 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:46:32.402 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:46:33.009 [conn130] end connection 127.0.0.3:58085 (3 connections now open) m31002| Wed Jan 16 18:46:33.009 [initandlisten] connection accepted from 127.0.0.3:50792 #132 (4 connections now open) m31000| Wed Jan 16 18:46:33.702 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:302156 151ms m31000| Wed Jan 16 18:46:34.402 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2173, "optime" : { "t" : 1358379994000, "i" : 5330 }, "optimeDate" : ISODate("2013-01-16T23:46:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2162, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2162, "optime" : { "t" : 1358379990000, "i" : 5250 }, "optimeDate" : ISODate("2013-01-16T23:46:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:33Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2172, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2159, "optime" : { "t" : 1358379990000, "i" : 5250 }, "optimeDate" : ISODate("2013-01-16T23:46:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:34Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2159, "optime" : { "t" : 1358379993000, "i" : 3308 }, "optimeDate" : ISODate("2013-01-16T23:46:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2159, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2171, "optime" : { "t" : 1358379994000, "i" : 4440 }, "optimeDate" : ISODate("2013-01-16T23:46:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:46:36.403 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:46:36.784 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:46:38.403 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2177, "optime" : { "t" : 1358379998000, "i" : 6805 }, "optimeDate" : ISODate("2013-01-16T23:46:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2166, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2166, "optime" : { "t" : 1358379994000, "i" : 5330 }, "optimeDate" : ISODate("2013-01-16T23:46:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:37Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2176, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2163, "optime" : { "t" : 1358379994000, "i" : 5330 }, "optimeDate" : ISODate("2013-01-16T23:46:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2163, "optime" : { "t" : 1358379997000, "i" : 2087 }, "optimeDate" : ISODate("2013-01-16T23:46:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2163, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2175, "optime" : { "t" : 1358379998000, "i" : 3572 }, "optimeDate" : ISODate("2013-01-16T23:46:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:46:39.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:46:40.403 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:46:41.785 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:46:42.404 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2181, "optime" : { "t" : 1358380002000, "i" : 8048 }, "optimeDate" : ISODate("2013-01-16T23:46:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2170, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2170, "optime" : { "t" : 1358380002000, "i" : 2674 }, "optimeDate" : ISODate("2013-01-16T23:46:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2180, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2167, "optime" : { "t" : 1358379998000, "i" : 6805 }, "optimeDate" : ISODate("2013-01-16T23:46:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:41Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2167, "optime" : { "t" : 1358380001000, "i" : 526 }, "optimeDate" : ISODate("2013-01-16T23:46:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2167, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2179, "optime" : { "t" : 1358380002000, "i" : 8048 }, "optimeDate" : ISODate("2013-01-16T23:46:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:46:44.404 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:46:46.404 [conn129] end connection 127.0.0.2:33034 (2 connections now open) m31001| Wed Jan 16 18:46:46.404 [initandlisten] connection accepted from 127.0.0.2:42685 #131 (3 connections now open) m31000| Wed Jan 16 18:46:46.405 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:46:46.786 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2186, "optime" : { "t" : 1358380007000, "i" : 202 }, "optimeDate" : ISODate("2013-01-16T23:46:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2175, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2175, "optime" : { "t" : 1358380002000, "i" : 8048 }, "optimeDate" : ISODate("2013-01-16T23:46:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:45Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2185, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2172, "optime" : { "t" : 1358380006000, "i" : 8585 }, "optimeDate" : ISODate("2013-01-16T23:46:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2172, "optime" : { "t" : 1358380002000, "i" : 8048 }, "optimeDate" : ISODate("2013-01-16T23:46:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2172, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2184, "optime" : { "t" : 1358380006000, "i" : 8585 }, "optimeDate" : ISODate("2013-01-16T23:46:46Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:46:48.405 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:46:49.862 [conn82] end connection 127.0.0.4:50299 (4 connections now open) m31000| Wed Jan 16 18:46:49.863 [initandlisten] connection accepted from 127.0.0.4:50427 #83 (5 connections now open) m31000| Wed Jan 16 18:46:50.405 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2190, "optime" : { "t" : 1358380011000, "i" : 1732 }, "optimeDate" : ISODate("2013-01-16T23:46:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2179, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2179, "optime" : { "t" : 1358380007000, "i" : 202 }, "optimeDate" : ISODate("2013-01-16T23:46:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:49Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2189, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2176, "optime" : { "t" : 1358380010000, "i" : 8258 }, "optimeDate" : ISODate("2013-01-16T23:46:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2176, "optime" : { "t" : 1358380007000, "i" : 202 }, "optimeDate" : ISODate("2013-01-16T23:46:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2176, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2188, "optime" : { "t" : 1358380011000, "i" : 1732 }, "optimeDate" : ISODate("2013-01-16T23:46:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:46:51.786 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:46:52.406 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:46:54.406 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2194, "optime" : { "t" : 1358380015000, "i" : 3004 }, "optimeDate" : ISODate("2013-01-16T23:46:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2183, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2183, "optime" : { "t" : 1358380014000, "i" : 5372 }, "optimeDate" : ISODate("2013-01-16T23:46:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2193, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2180, "optime" : { "t" : 1358380014000, "i" : 6870 }, "optimeDate" : ISODate("2013-01-16T23:46:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2180, "optime" : { "t" : 1358380011000, "i" : 1732 }, "optimeDate" : ISODate("2013-01-16T23:46:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2180, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2192, "optime" : { "t" : 1358380015000, "i" : 3004 }, "optimeDate" : ISODate("2013-01-16T23:46:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:46:56.406 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:46:56.787 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:46:58.407 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2198, "optime" : { "t" : 1358380019000, "i" : 4396 }, "optimeDate" : ISODate("2013-01-16T23:46:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2187, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2187, "optime" : { "t" : 1358380015000, "i" : 3004 }, "optimeDate" : ISODate("2013-01-16T23:46:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:57Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2197, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2184, "optime" : { "t" : 1358380018000, "i" : 5603 }, "optimeDate" : ISODate("2013-01-16T23:46:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:46:59Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:46:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2184, "optime" : { "t" : 1358380015000, "i" : 3004 }, "optimeDate" : ISODate("2013-01-16T23:46:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2184, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2196, "optime" : { "t" : 1358380019000, "i" : 4139 }, "optimeDate" : ISODate("2013-01-16T23:46:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:46:59.795 [conn131] end connection 127.0.0.2:36383 (3 connections now open) m31002| Wed Jan 16 18:47:00.296 [initandlisten] connection accepted from 127.0.0.2:33216 #133 (4 connections now open) m31000| Wed Jan 16 18:47:00.407 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:47:01.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:47:01.788 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:47:02.407 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:47:03.015 [conn132] end connection 127.0.0.3:50792 (3 connections now open) m31002| Wed Jan 16 18:47:03.015 [initandlisten] connection accepted from 127.0.0.3:33913 #134 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2202, "optime" : { "t" : 1358380023000, "i" : 5224 }, "optimeDate" : ISODate("2013-01-16T23:47:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2191, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2191, "optime" : { "t" : 1358380022000, "i" : 2422 }, "optimeDate" : ISODate("2013-01-16T23:47:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2201, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2188, "optime" : { "t" : 1358380023000, "i" : 80 }, "optimeDate" : ISODate("2013-01-16T23:47:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2188, "optime" : { "t" : 1358380019000, "i" : 4396 }, "optimeDate" : ISODate("2013-01-16T23:46:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2188, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:03Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2200, "optime" : { "t" : 1358380023000, "i" : 4640 }, "optimeDate" : ISODate("2013-01-16T23:47:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:47:04.288 [conn130] end connection 127.0.0.4:59086 (2 connections now open) m31000| Wed Jan 16 18:47:04.408 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:47:04.788 [initandlisten] connection accepted from 127.0.0.4:46083 #132 (3 connections now open) m31000| Wed Jan 16 18:47:06.408 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:47:07.288 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2206, "optime" : { "t" : 1358380027000, "i" : 6958 }, "optimeDate" : ISODate("2013-01-16T23:47:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2195, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2195, "optime" : { "t" : 1358380023000, "i" : 5224 }, "optimeDate" : ISODate("2013-01-16T23:47:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:05Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2205, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2192, "optime" : { "t" : 1358380027000, "i" : 173 }, "optimeDate" : ISODate("2013-01-16T23:47:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:07Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2192, "optime" : { "t" : 1358380023000, "i" : 5224 }, "optimeDate" : ISODate("2013-01-16T23:47:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2192, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2204, "optime" : { "t" : 1358380027000, "i" : 6937 }, "optimeDate" : ISODate("2013-01-16T23:47:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:47:08.409 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:47:10.409 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:47:11.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2210, "optime" : { "t" : 1358380031000, "i" : 7821 }, "optimeDate" : ISODate("2013-01-16T23:47:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2199, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2199, "optime" : { "t" : 1358380027000, "i" : 6958 }, "optimeDate" : ISODate("2013-01-16T23:47:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2209, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2196, "optime" : { "t" : 1358380030000, "i" : 2169 }, "optimeDate" : ISODate("2013-01-16T23:47:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2196, "optime" : { "t" : 1358380027000, "i" : 6958 }, "optimeDate" : ISODate("2013-01-16T23:47:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2196, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2208, "optime" : { "t" : 1358380031000, "i" : 7682 }, "optimeDate" : ISODate("2013-01-16T23:47:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:47:12.289 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:47:12.409 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:47:14.410 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2215, "optime" : { "t" : 1358380036000, "i" : 422 }, "optimeDate" : ISODate("2013-01-16T23:47:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2204, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2204, "optime" : { "t" : 1358380035000, "i" : 2404 }, "optimeDate" : ISODate("2013-01-16T23:47:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:15Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2214, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2201, "optime" : { "t" : 1358380035000, "i" : 76 }, "optimeDate" : ISODate("2013-01-16T23:47:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2201, "optime" : { "t" : 1358380035000, "i" : 7179 }, "optimeDate" : ISODate("2013-01-16T23:47:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2201, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:15Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2213, "optime" : { "t" : 1358380036000, "i" : 267 }, "optimeDate" : ISODate("2013-01-16T23:47:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:47:16.410 [conn131] end connection 127.0.0.2:42685 (2 connections now open) m31001| Wed Jan 16 18:47:16.410 [initandlisten] connection accepted from 127.0.0.2:50447 #133 (3 connections now open) m31000| Wed Jan 16 18:47:16.411 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:47:17.290 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:47:18.411 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:47:19.868 [conn83] end connection 127.0.0.4:50427 (4 connections now open) m31000| Wed Jan 16 18:47:19.868 [initandlisten] connection accepted from 127.0.0.4:56838 #84 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2219, "optime" : { "t" : 1358380040000, "i" : 2305 }, "optimeDate" : ISODate("2013-01-16T23:47:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2208, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2208, "optime" : { "t" : 1358380036000, "i" : 422 }, "optimeDate" : ISODate("2013-01-16T23:47:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:19Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:46:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2218, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2205, "optime" : { "t" : 1358380036000, "i" : 422 }, "optimeDate" : ISODate("2013-01-16T23:47:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:19Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2205, "optime" : { "t" : 1358380039000, "i" : 6526 }, "optimeDate" : ISODate("2013-01-16T23:47:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2205, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2217, "optime" : { "t" : 1358380040000, "i" : 2305 }, "optimeDate" : ISODate("2013-01-16T23:47:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:47:20.411 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:47:22.291 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:47:22.412 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2223, "optime" : { "t" : 1358380044000, "i" : 2949 }, "optimeDate" : ISODate("2013-01-16T23:47:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2212, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2212, "optime" : { "t" : 1358380040000, "i" : 2305 }, "optimeDate" : ISODate("2013-01-16T23:47:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:23Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2222, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2209, "optime" : { "t" : 1358380040000, "i" : 2305 }, "optimeDate" : ISODate("2013-01-16T23:47:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2209, "optime" : { "t" : 1358380043000, "i" : 5809 }, "optimeDate" : ISODate("2013-01-16T23:47:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2209, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2221, "optime" : { "t" : 1358380044000, "i" : 2949 }, "optimeDate" : ISODate("2013-01-16T23:47:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:47:24.412 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:47:24.451 [journal] old journal file will be removed: /data/db/testReplSet-0/journal/j._2 m31000| Wed Jan 16 18:47:26.412 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:47:27.291 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:47:28.413 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2227, "optime" : { "t" : 1358380048000, "i" : 4458 }, "optimeDate" : ISODate("2013-01-16T23:47:28Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2216, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2216, "optime" : { "t" : 1358380047000, "i" : 3610 }, "optimeDate" : ISODate("2013-01-16T23:47:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:27Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2226, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2213, "optime" : { "t" : 1358380044000, "i" : 2949 }, "optimeDate" : ISODate("2013-01-16T23:47:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:27Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:28Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2213, "optime" : { "t" : 1358380047000, "i" : 4655 }, "optimeDate" : ISODate("2013-01-16T23:47:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2213, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2225, "optime" : { "t" : 1358380048000, "i" : 4458 }, "optimeDate" : ISODate("2013-01-16T23:47:28Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:47:30.413 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:47:32.292 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:47:32.413 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2231, "optime" : { "t" : 1358380052000, "i" : 5999 }, "optimeDate" : ISODate("2013-01-16T23:47:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2220, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2220, "optime" : { "t" : 1358380048000, "i" : 4458 }, "optimeDate" : ISODate("2013-01-16T23:47:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:31Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2230, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2217, "optime" : { "t" : 1358380048000, "i" : 4458 }, "optimeDate" : ISODate("2013-01-16T23:47:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:32Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2217, "optime" : { "t" : 1358380051000, "i" : 2854 }, "optimeDate" : ISODate("2013-01-16T23:47:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2217, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2229, "optime" : { "t" : 1358380052000, "i" : 5999 }, "optimeDate" : ISODate("2013-01-16T23:47:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:47:33.021 [conn134] end connection 127.0.0.3:33913 (3 connections now open) m31002| Wed Jan 16 18:47:33.021 [initandlisten] connection accepted from 127.0.0.3:60065 #135 (4 connections now open) m31001| Wed Jan 16 18:47:33.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:47:34.414 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:47:36.414 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2235, "optime" : { "t" : 1358380056000, "i" : 7364 }, "optimeDate" : ISODate("2013-01-16T23:47:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2224, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2224, "optime" : { "t" : 1358380052000, "i" : 5999 }, "optimeDate" : ISODate("2013-01-16T23:47:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:35Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2234, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2221, "optime" : { "t" : 1358380052000, "i" : 5999 }, "optimeDate" : ISODate("2013-01-16T23:47:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2221, "optime" : { "t" : 1358380055000, "i" : 1511 }, "optimeDate" : ISODate("2013-01-16T23:47:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2221, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:35Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2233, "optime" : { "t" : 1358380056000, "i" : 7004 }, "optimeDate" : ISODate("2013-01-16T23:47:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:47:37.293 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:47:37.801 [conn133] end connection 127.0.0.2:33216 (3 connections now open) m31002| Wed Jan 16 18:47:38.301 [initandlisten] connection accepted from 127.0.0.2:38530 #136 (4 connections now open) m31000| Wed Jan 16 18:47:38.414 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:47:40.415 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2239, "optime" : { "t" : 1358380060000, "i" : 8448 }, "optimeDate" : ISODate("2013-01-16T23:47:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2228, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2228, "optime" : { "t" : 1358380060000, "i" : 7113 }, "optimeDate" : ISODate("2013-01-16T23:47:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2238, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2225, "optime" : { "t" : 1358380056000, "i" : 7364 }, "optimeDate" : ISODate("2013-01-16T23:47:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:39Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2225, "optime" : { "t" : 1358380059000, "i" : 93 }, "optimeDate" : ISODate("2013-01-16T23:47:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2225, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2237, "optime" : { "t" : 1358380060000, "i" : 8448 }, "optimeDate" : ISODate("2013-01-16T23:47:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:47:42.293 [conn132] end connection 127.0.0.4:46083 (2 connections now open) m31000| Wed Jan 16 18:47:42.415 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:47:42.793 [initandlisten] connection accepted from 127.0.0.4:56025 #134 (3 connections now open) m31002| Wed Jan 16 18:47:42.794 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:47:43.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:47:44.415 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2244, "optime" : { "t" : 1358380065000, "i" : 1578 }, "optimeDate" : ISODate("2013-01-16T23:47:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2233, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2233, "optime" : { "t" : 1358380060000, "i" : 8448 }, "optimeDate" : ISODate("2013-01-16T23:47:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:43Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2243, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2230, "optime" : { "t" : 1358380065000, "i" : 217 }, "optimeDate" : ISODate("2013-01-16T23:47:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2230, "optime" : { "t" : 1358380060000, "i" : 8448 }, "optimeDate" : ISODate("2013-01-16T23:47:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2230, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2242, "optime" : { "t" : 1358380065000, "i" : 1578 }, "optimeDate" : ISODate("2013-01-16T23:47:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:47:46.416 [conn133] end connection 127.0.0.2:50447 (2 connections now open) m31001| Wed Jan 16 18:47:46.416 [initandlisten] connection accepted from 127.0.0.2:51652 #135 (3 connections now open) m31000| Wed Jan 16 18:47:46.416 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:47:47.795 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:47:48.416 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:47:48.808 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:609929 304ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2248, "optime" : { "t" : 1358380069000, "i" : 4209 }, "optimeDate" : ISODate("2013-01-16T23:47:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2237, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2237, "optime" : { "t" : 1358380068000, "i" : 1239 }, "optimeDate" : ISODate("2013-01-16T23:47:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2247, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2234, "optime" : { "t" : 1358380069000, "i" : 616 }, "optimeDate" : ISODate("2013-01-16T23:47:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2234, "optime" : { "t" : 1358380065000, "i" : 1578 }, "optimeDate" : ISODate("2013-01-16T23:47:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2234, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2246, "optime" : { "t" : 1358380069000, "i" : 4209 }, "optimeDate" : ISODate("2013-01-16T23:47:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:47:49.873 [conn84] end connection 127.0.0.4:56838 (4 connections now open) m31000| Wed Jan 16 18:47:49.873 [initandlisten] connection accepted from 127.0.0.4:41200 #85 (5 connections now open) m31000| Wed Jan 16 18:47:50.417 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:47:52.417 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:47:52.795 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:47:53.257 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:226953 113ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2252, "optime" : { "t" : 1358380073000, "i" : 3546 }, "optimeDate" : ISODate("2013-01-16T23:47:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2241, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2241, "optime" : { "t" : 1358380073000, "i" : 2417 }, "optimeDate" : ISODate("2013-01-16T23:47:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2251, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2238, "optime" : { "t" : 1358380072000, "i" : 6454 }, "optimeDate" : ISODate("2013-01-16T23:47:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2238, "optime" : { "t" : 1358380069000, "i" : 4209 }, "optimeDate" : ISODate("2013-01-16T23:47:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:53Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2238, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:53Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2250, "optime" : { "t" : 1358380073000, "i" : 2510 }, "optimeDate" : ISODate("2013-01-16T23:47:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:47:54.417 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:47:56.418 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2256, "optime" : { "t" : 1358380077000, "i" : 4211 }, "optimeDate" : ISODate("2013-01-16T23:47:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2245, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2245, "optime" : { "t" : 1358380073000, "i" : 3546 }, "optimeDate" : ISODate("2013-01-16T23:47:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:55Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2255, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2242, "optime" : { "t" : 1358380077000, "i" : 144 }, "optimeDate" : ISODate("2013-01-16T23:47:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:47:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2242, "optime" : { "t" : 1358380073000, "i" : 3546 }, "optimeDate" : ISODate("2013-01-16T23:47:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2242, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:47:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2254, "optime" : { "t" : 1358380077000, "i" : 3770 }, "optimeDate" : ISODate("2013-01-16T23:47:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:47:57.796 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:47:58.418 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:48:00.419 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2260, "optime" : { "t" : 1358380081000, "i" : 5154 }, "optimeDate" : ISODate("2013-01-16T23:48:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2249, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2249, "optime" : { "t" : 1358380080000, "i" : 2948 }, "optimeDate" : ISODate("2013-01-16T23:48:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2259, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2246, "optime" : { "t" : 1358380080000, "i" : 4752 }, "optimeDate" : ISODate("2013-01-16T23:48:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2246, "optime" : { "t" : 1358380077000, "i" : 4211 }, "optimeDate" : ISODate("2013-01-16T23:47:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2246, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2258, "optime" : { "t" : 1358380081000, "i" : 4837 }, "optimeDate" : ISODate("2013-01-16T23:48:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:48:02.419 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:48:02.797 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:48:03.026 [conn135] end connection 127.0.0.3:60065 (3 connections now open) m31002| Wed Jan 16 18:48:03.026 [initandlisten] connection accepted from 127.0.0.3:45214 #137 (4 connections now open) m31000| Wed Jan 16 18:48:04.419 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:48:05.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2264, "optime" : { "t" : 1358380085000, "i" : 7055 }, "optimeDate" : ISODate("2013-01-16T23:48:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2253, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2253, "optime" : { "t" : 1358380081000, "i" : 5154 }, "optimeDate" : ISODate("2013-01-16T23:48:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:03Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2263, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2250, "optime" : { "t" : 1358380085000, "i" : 616 }, "optimeDate" : ISODate("2013-01-16T23:48:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:05Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2250, "optime" : { "t" : 1358380081000, "i" : 5154 }, "optimeDate" : ISODate("2013-01-16T23:48:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:05Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2250, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2262, "optime" : { "t" : 1358380085000, "i" : 6937 }, "optimeDate" : ISODate("2013-01-16T23:48:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:48:06.420 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:48:07.797 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:48:08.420 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:48:09.475 [journal] old journal file will be removed: /data/db/testReplSet-2/journal/j._2 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2268, "optime" : { "t" : 1358380089000, "i" : 8500 }, "optimeDate" : ISODate("2013-01-16T23:48:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2257, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2257, "optime" : { "t" : 1358380085000, "i" : 7055 }, "optimeDate" : ISODate("2013-01-16T23:48:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:09Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2267, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2254, "optime" : { "t" : 1358380088000, "i" : 1499 }, "optimeDate" : ISODate("2013-01-16T23:48:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2254, "optime" : { "t" : 1358380089000, "i" : 7849 }, "optimeDate" : ISODate("2013-01-16T23:48:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2254, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:09Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2266, "optime" : { "t" : 1358380089000, "i" : 8500 }, "optimeDate" : ISODate("2013-01-16T23:48:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:48:10.421 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:48:12.421 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:48:12.798 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2273, "optime" : { "t" : 1358380094000, "i" : 992 }, "optimeDate" : ISODate("2013-01-16T23:48:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2262, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2262, "optime" : { "t" : 1358380093000, "i" : 3064 }, "optimeDate" : ISODate("2013-01-16T23:48:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:13Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2272, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2259, "optime" : { "t" : 1358380093000, "i" : 43 }, "optimeDate" : ISODate("2013-01-16T23:48:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2259, "optime" : { "t" : 1358380093000, "i" : 8501 }, "optimeDate" : ISODate("2013-01-16T23:48:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2259, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:13Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2271, "optime" : { "t" : 1358380094000, "i" : 992 }, "optimeDate" : ISODate("2013-01-16T23:48:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:48:14.421 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:48:15.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:48:15.806 [conn136] end connection 127.0.0.2:38530 (3 connections now open) m31002| Wed Jan 16 18:48:16.306 [initandlisten] connection accepted from 127.0.0.2:41106 #138 (4 connections now open) m31001| Wed Jan 16 18:48:16.422 [conn135] end connection 127.0.0.2:51652 (2 connections now open) m31001| Wed Jan 16 18:48:16.422 [initandlisten] connection accepted from 127.0.0.2:58031 #136 (3 connections now open) m31000| Wed Jan 16 18:48:16.422 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:48:17.799 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2277, "optime" : { "t" : 1358380098000, "i" : 1977 }, "optimeDate" : ISODate("2013-01-16T23:48:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2266, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2266, "optime" : { "t" : 1358380094000, "i" : 992 }, "optimeDate" : ISODate("2013-01-16T23:48:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:17Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2276, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2263, "optime" : { "t" : 1358380094000, "i" : 992 }, "optimeDate" : ISODate("2013-01-16T23:48:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:17Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2263, "optime" : { "t" : 1358380097000, "i" : 7246 }, "optimeDate" : ISODate("2013-01-16T23:48:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2263, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2275, "optime" : { "t" : 1358380098000, "i" : 1687 }, "optimeDate" : ISODate("2013-01-16T23:48:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:48:18.422 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:48:19.878 [conn85] end connection 127.0.0.4:41200 (4 connections now open) m31000| Wed Jan 16 18:48:19.879 [initandlisten] connection accepted from 127.0.0.4:56678 #86 (5 connections now open) m31001| Wed Jan 16 18:48:20.299 [conn134] end connection 127.0.0.4:56025 (2 connections now open) m31000| Wed Jan 16 18:48:20.423 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:48:20.799 [initandlisten] connection accepted from 127.0.0.4:47021 #137 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2281, "optime" : { "t" : 1358380102000, "i" : 2985 }, "optimeDate" : ISODate("2013-01-16T23:48:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2270, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2270, "optime" : { "t" : 1358380101000, "i" : 582 }, "optimeDate" : ISODate("2013-01-16T23:48:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:21Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:47:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2280, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2267, "optime" : { "t" : 1358380098000, "i" : 1977 }, "optimeDate" : ISODate("2013-01-16T23:48:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2267, "optime" : { "t" : 1358380101000, "i" : 5732 }, "optimeDate" : ISODate("2013-01-16T23:48:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2267, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:21Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2279, "optime" : { "t" : 1358380102000, "i" : 2938 }, "optimeDate" : ISODate("2013-01-16T23:48:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:48:22.423 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:48:23.300 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:48:24.424 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:48:26.424 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2285, "optime" : { "t" : 1358380106000, "i" : 4135 }, "optimeDate" : ISODate("2013-01-16T23:48:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2274, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2274, "optime" : { "t" : 1358380106000, "i" : 2642 }, "optimeDate" : ISODate("2013-01-16T23:48:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2284, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2271, "optime" : { "t" : 1358380102000, "i" : 2985 }, "optimeDate" : ISODate("2013-01-16T23:48:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:25Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2271, "optime" : { "t" : 1358380105000, "i" : 4812 }, "optimeDate" : ISODate("2013-01-16T23:48:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2271, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2283, "optime" : { "t" : 1358380106000, "i" : 4135 }, "optimeDate" : ISODate("2013-01-16T23:48:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:48:28.300 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:48:28.425 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:48:30.425 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2289, "optime" : { "t" : 1358380110000, "i" : 5736 }, "optimeDate" : ISODate("2013-01-16T23:48:30Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2278, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2278, "optime" : { "t" : 1358380106000, "i" : 4135 }, "optimeDate" : ISODate("2013-01-16T23:48:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:29Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2288, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2275, "optime" : { "t" : 1358380106000, "i" : 4135 }, "optimeDate" : ISODate("2013-01-16T23:48:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2275, "optime" : { "t" : 1358380109000, "i" : 3683 }, "optimeDate" : ISODate("2013-01-16T23:48:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2275, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:29Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2287, "optime" : { "t" : 1358380110000, "i" : 5731 }, "optimeDate" : ISODate("2013-01-16T23:48:30Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:48:32.425 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:48:33.031 [conn137] end connection 127.0.0.3:45214 (3 connections now open) m31002| Wed Jan 16 18:48:33.031 [initandlisten] connection accepted from 127.0.0.3:37074 #139 (4 connections now open) m31002| Wed Jan 16 18:48:33.301 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:48:34.425 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2293, "optime" : { "t" : 1358380114000, "i" : 6428 }, "optimeDate" : ISODate("2013-01-16T23:48:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2282, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2282, "optime" : { "t" : 1358380113000, "i" : 1563 }, "optimeDate" : ISODate("2013-01-16T23:48:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:33Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2292, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2279, "optime" : { "t" : 1358380110000, "i" : 5736 }, "optimeDate" : ISODate("2013-01-16T23:48:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:33Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2279, "optime" : { "t" : 1358380113000, "i" : 2627 }, "optimeDate" : ISODate("2013-01-16T23:48:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2279, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2291, "optime" : { "t" : 1358380114000, "i" : 6428 }, "optimeDate" : ISODate("2013-01-16T23:48:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:48:36.426 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:48:37.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:48:38.302 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:48:38.426 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2297, "optime" : { "t" : 1358380118000, "i" : 8327 }, "optimeDate" : ISODate("2013-01-16T23:48:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2286, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2286, "optime" : { "t" : 1358380118000, "i" : 6977 }, "optimeDate" : ISODate("2013-01-16T23:48:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2296, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2283, "optime" : { "t" : 1358380114000, "i" : 6428 }, "optimeDate" : ISODate("2013-01-16T23:48:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:38Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2283, "optime" : { "t" : 1358380117000, "i" : 1078 }, "optimeDate" : ISODate("2013-01-16T23:48:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2283, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2295, "optime" : { "t" : 1358380118000, "i" : 8312 }, "optimeDate" : ISODate("2013-01-16T23:48:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:48:40.427 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:48:42.427 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2302, "optime" : { "t" : 1358380123000, "i" : 1233 }, "optimeDate" : ISODate("2013-01-16T23:48:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2291, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2291, "optime" : { "t" : 1358380118000, "i" : 8327 }, "optimeDate" : ISODate("2013-01-16T23:48:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:41Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2301, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2288, "optime" : { "t" : 1358380122000, "i" : 8535 }, "optimeDate" : ISODate("2013-01-16T23:48:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2288, "optime" : { "t" : 1358380118000, "i" : 8327 }, "optimeDate" : ISODate("2013-01-16T23:48:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2288, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:43Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2300, "optime" : { "t" : 1358380123000, "i" : 1233 }, "optimeDate" : ISODate("2013-01-16T23:48:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:48:43.302 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:48:44.427 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:48:46.427 [conn136] end connection 127.0.0.2:58031 (2 connections now open) m31001| Wed Jan 16 18:48:46.427 [initandlisten] connection accepted from 127.0.0.2:35868 #138 (3 connections now open) m31000| Wed Jan 16 18:48:46.428 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2306, "optime" : { "t" : 1358380127000, "i" : 1967 }, "optimeDate" : ISODate("2013-01-16T23:48:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2295, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2295, "optime" : { "t" : 1358380126000, "i" : 1287 }, "optimeDate" : ISODate("2013-01-16T23:48:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2305, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2292, "optime" : { "t" : 1358380126000, "i" : 8032 }, "optimeDate" : ISODate("2013-01-16T23:48:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2292, "optime" : { "t" : 1358380123000, "i" : 1233 }, "optimeDate" : ISODate("2013-01-16T23:48:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2292, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2304, "optime" : { "t" : 1358380127000, "i" : 1302 }, "optimeDate" : ISODate("2013-01-16T23:48:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:48:47.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:48:48.303 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:48:48.428 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:48:49.883 [conn86] end connection 127.0.0.4:56678 (4 connections now open) m31000| Wed Jan 16 18:48:49.884 [initandlisten] connection accepted from 127.0.0.4:42807 #87 (5 connections now open) m31000| Wed Jan 16 18:48:50.428 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2310, "optime" : { "t" : 1358380131000, "i" : 3120 }, "optimeDate" : ISODate("2013-01-16T23:48:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2299, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2299, "optime" : { "t" : 1358380131000, "i" : 2595 }, "optimeDate" : ISODate("2013-01-16T23:48:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2309, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2296, "optime" : { "t" : 1358380131000, "i" : 142 }, "optimeDate" : ISODate("2013-01-16T23:48:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2296, "optime" : { "t" : 1358380127000, "i" : 1967 }, "optimeDate" : ISODate("2013-01-16T23:48:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:51Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2296, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2308, "optime" : { "t" : 1358380131000, "i" : 3120 }, "optimeDate" : ISODate("2013-01-16T23:48:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:48:52.429 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:48:53.304 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:48:53.811 [conn138] end connection 127.0.0.2:41106 (3 connections now open) m31002| Wed Jan 16 18:48:54.311 [initandlisten] connection accepted from 127.0.0.2:46095 #140 (4 connections now open) m31000| Wed Jan 16 18:48:54.429 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2314, "optime" : { "t" : 1358380135000, "i" : 4788 }, "optimeDate" : ISODate("2013-01-16T23:48:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2303, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2303, "optime" : { "t" : 1358380131000, "i" : 3120 }, "optimeDate" : ISODate("2013-01-16T23:48:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2313, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2300, "optime" : { "t" : 1358380135000, "i" : 72 }, "optimeDate" : ISODate("2013-01-16T23:48:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2300, "optime" : { "t" : 1358380131000, "i" : 3120 }, "optimeDate" : ISODate("2013-01-16T23:48:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2300, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2312, "optime" : { "t" : 1358380135000, "i" : 4559 }, "optimeDate" : ISODate("2013-01-16T23:48:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:48:56.429 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:48:58.304 [conn137] end connection 127.0.0.4:47021 (2 connections now open) m31000| Wed Jan 16 18:48:58.430 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:48:58.805 [initandlisten] connection accepted from 127.0.0.4:48095 #139 (3 connections now open) m31002| Wed Jan 16 18:48:58.805 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2318, "optime" : { "t" : 1358380139000, "i" : 5901 }, "optimeDate" : ISODate("2013-01-16T23:48:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2307, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2307, "optime" : { "t" : 1358380139000, "i" : 2661 }, "optimeDate" : ISODate("2013-01-16T23:48:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2317, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2304, "optime" : { "t" : 1358380139000, "i" : 195 }, "optimeDate" : ISODate("2013-01-16T23:48:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:48:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2304, "optime" : { "t" : 1358380135000, "i" : 4788 }, "optimeDate" : ISODate("2013-01-16T23:48:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:59Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2304, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:48:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2316, "optime" : { "t" : 1358380139000, "i" : 5059 }, "optimeDate" : ISODate("2013-01-16T23:48:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:49:00.430 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:49:02.430 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:49:03.037 [conn139] end connection 127.0.0.3:37074 (3 connections now open) m31002| Wed Jan 16 18:49:03.037 [initandlisten] connection accepted from 127.0.0.3:39874 #141 (4 connections now open) m31002| Wed Jan 16 18:49:03.805 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2322, "optime" : { "t" : 1358380143000, "i" : 7492 }, "optimeDate" : ISODate("2013-01-16T23:49:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2311, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2311, "optime" : { "t" : 1358380139000, "i" : 5901 }, "optimeDate" : ISODate("2013-01-16T23:48:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:01Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2321, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2308, "optime" : { "t" : 1358380143000, "i" : 192 }, "optimeDate" : ISODate("2013-01-16T23:49:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:03Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2308, "optime" : { "t" : 1358380139000, "i" : 5901 }, "optimeDate" : ISODate("2013-01-16T23:48:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2308, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2320, "optime" : { "t" : 1358380143000, "i" : 7492 }, "optimeDate" : ISODate("2013-01-16T23:49:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:49:04.431 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:49:06.431 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2326, "optime" : { "t" : 1358380147000, "i" : 8744 }, "optimeDate" : ISODate("2013-01-16T23:49:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2315, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2315, "optime" : { "t" : 1358380143000, "i" : 7492 }, "optimeDate" : ISODate("2013-01-16T23:49:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:07Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2325, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2312, "optime" : { "t" : 1358380147000, "i" : 226 }, "optimeDate" : ISODate("2013-01-16T23:49:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2312, "optime" : { "t" : 1358380147000, "i" : 7573 }, "optimeDate" : ISODate("2013-01-16T23:49:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2312, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:07Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2324, "optime" : { "t" : 1358380147000, "i" : 8744 }, "optimeDate" : ISODate("2013-01-16T23:49:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:49:08.432 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:49:08.806 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:49:09.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:49:10.432 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2331, "optime" : { "t" : 1358380152000, "i" : 1017 }, "optimeDate" : ISODate("2013-01-16T23:49:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2320, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2320, "optime" : { "t" : 1358380151000, "i" : 6952 }, "optimeDate" : ISODate("2013-01-16T23:49:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:11Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2330, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2317, "optime" : { "t" : 1358380151000, "i" : 261 }, "optimeDate" : ISODate("2013-01-16T23:49:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:11Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2317, "optime" : { "t" : 1358380151000, "i" : 7597 }, "optimeDate" : ISODate("2013-01-16T23:49:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2317, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2329, "optime" : { "t" : 1358380152000, "i" : 1017 }, "optimeDate" : ISODate("2013-01-16T23:49:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:49:12.433 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:49:13.807 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:49:14.433 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2335, "optime" : { "t" : 1358380156000, "i" : 1951 }, "optimeDate" : ISODate("2013-01-16T23:49:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2324, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2324, "optime" : { "t" : 1358380152000, "i" : 1017 }, "optimeDate" : ISODate("2013-01-16T23:49:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:15Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2334, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2321, "optime" : { "t" : 1358380152000, "i" : 1017 }, "optimeDate" : ISODate("2013-01-16T23:49:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2321, "optime" : { "t" : 1358380155000, "i" : 6877 }, "optimeDate" : ISODate("2013-01-16T23:49:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2321, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:15Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2333, "optime" : { "t" : 1358380156000, "i" : 1787 }, "optimeDate" : ISODate("2013-01-16T23:49:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:49:16.433 [conn138] end connection 127.0.0.2:35868 (2 connections now open) m31001| Wed Jan 16 18:49:16.433 [initandlisten] connection accepted from 127.0.0.2:55567 #140 (3 connections now open) m31000| Wed Jan 16 18:49:16.434 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:49:18.434 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:49:18.807 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:49:19.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:49:19.889 [conn87] end connection 127.0.0.4:42807 (4 connections now open) m31000| Wed Jan 16 18:49:19.889 [initandlisten] connection accepted from 127.0.0.4:33961 #88 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2339, "optime" : { "t" : 1358380160000, "i" : 2829 }, "optimeDate" : ISODate("2013-01-16T23:49:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2328, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2328, "optime" : { "t" : 1358380159000, "i" : 187 }, "optimeDate" : ISODate("2013-01-16T23:49:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:19Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2338, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2325, "optime" : { "t" : 1358380156000, "i" : 1951 }, "optimeDate" : ISODate("2013-01-16T23:49:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2325, "optime" : { "t" : 1358380159000, "i" : 6498 }, "optimeDate" : ISODate("2013-01-16T23:49:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2325, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:19Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2337, "optime" : { "t" : 1358380160000, "i" : 2693 }, "optimeDate" : ISODate("2013-01-16T23:49:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:49:20.434 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:49:22.435 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:49:23.647 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:310871 155ms m31002| Wed Jan 16 18:49:23.808 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2343, "optime" : { "t" : 1358380164000, "i" : 3271 }, "optimeDate" : ISODate("2013-01-16T23:49:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2332, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2332, "optime" : { "t" : 1358380164000, "i" : 2499 }, "optimeDate" : ISODate("2013-01-16T23:49:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2342, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2329, "optime" : { "t" : 1358380160000, "i" : 2829 }, "optimeDate" : ISODate("2013-01-16T23:49:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:23Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2329, "optime" : { "t" : 1358380163000, "i" : 5551 }, "optimeDate" : ISODate("2013-01-16T23:49:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2329, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2341, "optime" : { "t" : 1358380164000, "i" : 3160 }, "optimeDate" : ISODate("2013-01-16T23:49:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:49:24.435 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:49:26.436 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:49:28.436 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2347, "optime" : { "t" : 1358380168000, "i" : 4231 }, "optimeDate" : ISODate("2013-01-16T23:49:28Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2336, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2336, "optime" : { "t" : 1358380164000, "i" : 3271 }, "optimeDate" : ISODate("2013-01-16T23:49:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:27Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:48:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2346, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2333, "optime" : { "t" : 1358380164000, "i" : 3271 }, "optimeDate" : ISODate("2013-01-16T23:49:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:28Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2333, "optime" : { "t" : 1358380167000, "i" : 4580 }, "optimeDate" : ISODate("2013-01-16T23:49:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2333, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:27Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2345, "optime" : { "t" : 1358380168000, "i" : 4231 }, "optimeDate" : ISODate("2013-01-16T23:49:28Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:49:28.809 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:49:30.436 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:49:31.816 [conn140] end connection 127.0.0.2:46095 (3 connections now open) m31002| Wed Jan 16 18:49:32.316 [initandlisten] connection accepted from 127.0.0.2:60165 #142 (4 connections now open) m31000| Wed Jan 16 18:49:32.436 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2351, "optime" : { "t" : 1358380172000, "i" : 6098 }, "optimeDate" : ISODate("2013-01-16T23:49:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2340, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2340, "optime" : { "t" : 1358380172000, "i" : 2777 }, "optimeDate" : ISODate("2013-01-16T23:49:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2350, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2337, "optime" : { "t" : 1358380168000, "i" : 4231 }, "optimeDate" : ISODate("2013-01-16T23:49:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:31Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2337, "optime" : { "t" : 1358380171000, "i" : 3059 }, "optimeDate" : ISODate("2013-01-16T23:49:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2337, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2349, "optime" : { "t" : 1358380172000, "i" : 6098 }, "optimeDate" : ISODate("2013-01-16T23:49:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:49:33.042 [conn141] end connection 127.0.0.3:39874 (3 connections now open) m31002| Wed Jan 16 18:49:33.042 [initandlisten] connection accepted from 127.0.0.3:50565 #143 (4 connections now open) m31002| Wed Jan 16 18:49:33.810 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:49:34.437 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:49:36.310 [conn139] end connection 127.0.0.4:48095 (2 connections now open) m31000| Wed Jan 16 18:49:36.437 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:49:36.810 [initandlisten] connection accepted from 127.0.0.4:59090 #141 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2355, "optime" : { "t" : 1358380176000, "i" : 7814 }, "optimeDate" : ISODate("2013-01-16T23:49:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2344, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2344, "optime" : { "t" : 1358380172000, "i" : 6098 }, "optimeDate" : ISODate("2013-01-16T23:49:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:35Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2354, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2341, "optime" : { "t" : 1358380172000, "i" : 6098 }, "optimeDate" : ISODate("2013-01-16T23:49:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:36Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2341, "optime" : { "t" : 1358380175000, "i" : 1633 }, "optimeDate" : ISODate("2013-01-16T23:49:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2341, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2353, "optime" : { "t" : 1358380176000, "i" : 7814 }, "optimeDate" : ISODate("2013-01-16T23:49:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:49:38.437 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:49:39.311 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:49:40.438 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2360, "optime" : { "t" : 1358380181000, "i" : 722 }, "optimeDate" : ISODate("2013-01-16T23:49:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2349, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2349, "optime" : { "t" : 1358380176000, "i" : 7814 }, "optimeDate" : ISODate("2013-01-16T23:49:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:39Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2359, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2346, "optime" : { "t" : 1358380181000, "i" : 308 }, "optimeDate" : ISODate("2013-01-16T23:49:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:41Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2346, "optime" : { "t" : 1358380179000, "i" : 129 }, "optimeDate" : ISODate("2013-01-16T23:49:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2346, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:41Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2358, "optime" : { "t" : 1358380181000, "i" : 722 }, "optimeDate" : ISODate("2013-01-16T23:49:41Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:49:41.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:49:42.438 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:49:44.311 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:49:44.438 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2364, "optime" : { "t" : 1358380185000, "i" : 1659 }, "optimeDate" : ISODate("2013-01-16T23:49:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2353, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2353, "optime" : { "t" : 1358380184000, "i" : 6633 }, "optimeDate" : ISODate("2013-01-16T23:49:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2363, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2350, "optime" : { "t" : 1358380185000, "i" : 44 }, "optimeDate" : ISODate("2013-01-16T23:49:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2350, "optime" : { "t" : 1358380181000, "i" : 722 }, "optimeDate" : ISODate("2013-01-16T23:49:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2350, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2362, "optime" : { "t" : 1358380185000, "i" : 1402 }, "optimeDate" : ISODate("2013-01-16T23:49:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:49:46.439 [conn140] end connection 127.0.0.2:55567 (2 connections now open) m31001| Wed Jan 16 18:49:46.439 [initandlisten] connection accepted from 127.0.0.2:50762 #142 (3 connections now open) m31000| Wed Jan 16 18:49:46.439 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:49:48.440 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:49:49.312 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2368, "optime" : { "t" : 1358380189000, "i" : 3134 }, "optimeDate" : ISODate("2013-01-16T23:49:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2357, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2357, "optime" : { "t" : 1358380185000, "i" : 1659 }, "optimeDate" : ISODate("2013-01-16T23:49:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:47Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2367, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2354, "optime" : { "t" : 1358380189000, "i" : 85 }, "optimeDate" : ISODate("2013-01-16T23:49:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:49Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2354, "optime" : { "t" : 1358380185000, "i" : 1659 }, "optimeDate" : ISODate("2013-01-16T23:49:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2354, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2366, "optime" : { "t" : 1358380189000, "i" : 3134 }, "optimeDate" : ISODate("2013-01-16T23:49:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:49:49.894 [conn88] end connection 127.0.0.4:33961 (4 connections now open) m31000| Wed Jan 16 18:49:49.894 [initandlisten] connection accepted from 127.0.0.4:46270 #89 (5 connections now open) m31000| Wed Jan 16 18:49:50.440 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:49:51.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:49:52.440 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:49:53.123 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:210914 105ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2372, "optime" : { "t" : 1358380193000, "i" : 5186 }, "optimeDate" : ISODate("2013-01-16T23:49:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2361, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2361, "optime" : { "t" : 1358380189000, "i" : 3134 }, "optimeDate" : ISODate("2013-01-16T23:49:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2371, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2358, "optime" : { "t" : 1358380193000, "i" : 104 }, "optimeDate" : ISODate("2013-01-16T23:49:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2358, "optime" : { "t" : 1358380189000, "i" : 3134 }, "optimeDate" : ISODate("2013-01-16T23:49:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2358, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:53Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2370, "optime" : { "t" : 1358380193000, "i" : 4629 }, "optimeDate" : ISODate("2013-01-16T23:49:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:49:54.313 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:49:54.441 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:49:56.441 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2376, "optime" : { "t" : 1358380197000, "i" : 5643 }, "optimeDate" : ISODate("2013-01-16T23:49:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2365, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2365, "optime" : { "t" : 1358380197000, "i" : 2842 }, "optimeDate" : ISODate("2013-01-16T23:49:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2375, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2362, "optime" : { "t" : 1358380197000, "i" : 44 }, "optimeDate" : ISODate("2013-01-16T23:49:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:49:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2362, "optime" : { "t" : 1358380193000, "i" : 5186 }, "optimeDate" : ISODate("2013-01-16T23:49:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:57Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2362, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2374, "optime" : { "t" : 1358380197000, "i" : 5643 }, "optimeDate" : ISODate("2013-01-16T23:49:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:49:58.441 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:49:59.313 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:50:00.442 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2380, "optime" : { "t" : 1358380201000, "i" : 7001 }, "optimeDate" : ISODate("2013-01-16T23:50:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2369, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2369, "optime" : { "t" : 1358380197000, "i" : 5643 }, "optimeDate" : ISODate("2013-01-16T23:49:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:49:59Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2379, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2366, "optime" : { "t" : 1358380201000, "i" : 31 }, "optimeDate" : ISODate("2013-01-16T23:50:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2366, "optime" : { "t" : 1358380197000, "i" : 5643 }, "optimeDate" : ISODate("2013-01-16T23:49:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2366, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:49:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2378, "optime" : { "t" : 1358380201000, "i" : 7001 }, "optimeDate" : ISODate("2013-01-16T23:50:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:50:02.442 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:50:03.048 [conn143] end connection 127.0.0.3:50565 (3 connections now open) m31002| Wed Jan 16 18:50:03.048 [initandlisten] connection accepted from 127.0.0.3:38007 #144 (4 connections now open) m31002| Wed Jan 16 18:50:04.314 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:50:04.443 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2384, "optime" : { "t" : 1358380205000, "i" : 8476 }, "optimeDate" : ISODate("2013-01-16T23:50:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2373, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2373, "optime" : { "t" : 1358380204000, "i" : 89 }, "optimeDate" : ISODate("2013-01-16T23:50:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:05Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2383, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2370, "optime" : { "t" : 1358380204000, "i" : 1524 }, "optimeDate" : ISODate("2013-01-16T23:50:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2370, "optime" : { "t" : 1358380205000, "i" : 8454 }, "optimeDate" : ISODate("2013-01-16T23:50:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2370, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:05Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2382, "optime" : { "t" : 1358380205000, "i" : 7710 }, "optimeDate" : ISODate("2013-01-16T23:50:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:50:06.443 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:50:08.444 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:50:09.315 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:50:09.822 [conn142] end connection 127.0.0.2:60165 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2389, "optime" : { "t" : 1358380210000, "i" : 507 }, "optimeDate" : ISODate("2013-01-16T23:50:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2378, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2378, "optime" : { "t" : 1358380205000, "i" : 8476 }, "optimeDate" : ISODate("2013-01-16T23:50:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:09Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2388, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2375, "optime" : { "t" : 1358380208000, "i" : 629 }, "optimeDate" : ISODate("2013-01-16T23:50:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:09Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2375, "optime" : { "t" : 1358380209000, "i" : 8183 }, "optimeDate" : ISODate("2013-01-16T23:50:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2375, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2387, "optime" : { "t" : 1358380210000, "i" : 308 }, "optimeDate" : ISODate("2013-01-16T23:50:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:50:10.322 [initandlisten] connection accepted from 127.0.0.2:44445 #145 (4 connections now open) m31000| Wed Jan 16 18:50:10.444 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:50:12.444 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:50:13.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2393, "optime" : { "t" : 1358380214000, "i" : 1966 }, "optimeDate" : ISODate("2013-01-16T23:50:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2382, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2382, "optime" : { "t" : 1358380210000, "i" : 507 }, "optimeDate" : ISODate("2013-01-16T23:50:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:13Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2392, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2379, "optime" : { "t" : 1358380210000, "i" : 507 }, "optimeDate" : ISODate("2013-01-16T23:50:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2379, "optime" : { "t" : 1358380213000, "i" : 6897 }, "optimeDate" : ISODate("2013-01-16T23:50:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2379, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:13Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2391, "optime" : { "t" : 1358380214000, "i" : 1966 }, "optimeDate" : ISODate("2013-01-16T23:50:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:50:14.315 [conn141] end connection 127.0.0.4:59090 (2 connections now open) m31000| Wed Jan 16 18:50:14.445 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:50:14.815 [initandlisten] connection accepted from 127.0.0.4:44518 #143 (3 connections now open) m31002| Wed Jan 16 18:50:14.816 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:50:16.445 [conn142] end connection 127.0.0.2:50762 (2 connections now open) m31001| Wed Jan 16 18:50:16.445 [initandlisten] connection accepted from 127.0.0.2:54058 #144 (3 connections now open) m31000| Wed Jan 16 18:50:16.445 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2397, "optime" : { "t" : 1358380218000, "i" : 3287 }, "optimeDate" : ISODate("2013-01-16T23:50:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2386, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2386, "optime" : { "t" : 1358380217000, "i" : 4570 }, "optimeDate" : ISODate("2013-01-16T23:50:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:17Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2396, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2383, "optime" : { "t" : 1358380214000, "i" : 1966 }, "optimeDate" : ISODate("2013-01-16T23:50:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:17Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2383, "optime" : { "t" : 1358380217000, "i" : 5523 }, "optimeDate" : ISODate("2013-01-16T23:50:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2383, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2395, "optime" : { "t" : 1358380218000, "i" : 3287 }, "optimeDate" : ISODate("2013-01-16T23:50:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:50:18.446 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:50:19.816 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:50:19.899 [conn89] end connection 127.0.0.4:46270 (4 connections now open) m31000| Wed Jan 16 18:50:19.900 [initandlisten] connection accepted from 127.0.0.4:37063 #90 (5 connections now open) m31000| Wed Jan 16 18:50:20.446 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:50:22.446 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2401, "optime" : { "t" : 1358380222000, "i" : 4431 }, "optimeDate" : ISODate("2013-01-16T23:50:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2390, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2390, "optime" : { "t" : 1358380218000, "i" : 3287 }, "optimeDate" : ISODate("2013-01-16T23:50:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:21Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2400, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2387, "optime" : { "t" : 1358380218000, "i" : 3287 }, "optimeDate" : ISODate("2013-01-16T23:50:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:22Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2387, "optime" : { "t" : 1358380221000, "i" : 4462 }, "optimeDate" : ISODate("2013-01-16T23:50:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2387, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2399, "optime" : { "t" : 1358380222000, "i" : 4431 }, "optimeDate" : ISODate("2013-01-16T23:50:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:50:23.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:50:24.447 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:50:24.817 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:50:26.447 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2405, "optime" : { "t" : 1358380226000, "i" : 5214 }, "optimeDate" : ISODate("2013-01-16T23:50:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2394, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2394, "optime" : { "t" : 1358380222000, "i" : 4431 }, "optimeDate" : ISODate("2013-01-16T23:50:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:25Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2404, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2391, "optime" : { "t" : 1358380222000, "i" : 4431 }, "optimeDate" : ISODate("2013-01-16T23:50:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2391, "optime" : { "t" : 1358380225000, "i" : 3777 }, "optimeDate" : ISODate("2013-01-16T23:50:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2391, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2403, "optime" : { "t" : 1358380226000, "i" : 4929 }, "optimeDate" : ISODate("2013-01-16T23:50:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:50:28.447 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:50:29.818 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:50:30.448 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2409, "optime" : { "t" : 1358380230000, "i" : 6985 }, "optimeDate" : ISODate("2013-01-16T23:50:30Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2398, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2398, "optime" : { "t" : 1358380230000, "i" : 3218 }, "optimeDate" : ISODate("2013-01-16T23:50:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2408, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2395, "optime" : { "t" : 1358380226000, "i" : 5214 }, "optimeDate" : ISODate("2013-01-16T23:50:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:29Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2395, "optime" : { "t" : 1358380229000, "i" : 2486 }, "optimeDate" : ISODate("2013-01-16T23:50:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2395, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2407, "optime" : { "t" : 1358380230000, "i" : 6782 }, "optimeDate" : ISODate("2013-01-16T23:50:30Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:50:32.448 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:50:33.053 [conn144] end connection 127.0.0.3:38007 (3 connections now open) m31002| Wed Jan 16 18:50:33.054 [initandlisten] connection accepted from 127.0.0.3:34230 #146 (4 connections now open) m31000| Wed Jan 16 18:50:34.448 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:50:34.818 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2413, "optime" : { "t" : 1358380234000, "i" : 8285 }, "optimeDate" : ISODate("2013-01-16T23:50:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2402, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2402, "optime" : { "t" : 1358380230000, "i" : 6985 }, "optimeDate" : ISODate("2013-01-16T23:50:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:33Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2412, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2399, "optime" : { "t" : 1358380230000, "i" : 6985 }, "optimeDate" : ISODate("2013-01-16T23:50:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:34Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2399, "optime" : { "t" : 1358380233000, "i" : 608 }, "optimeDate" : ISODate("2013-01-16T23:50:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2399, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2411, "optime" : { "t" : 1358380234000, "i" : 7536 }, "optimeDate" : ISODate("2013-01-16T23:50:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:50:36.449 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:50:38.449 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:50:38.702 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:460383 230ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2417, "optime" : { "t" : 1358380238000, "i" : 9447 }, "optimeDate" : ISODate("2013-01-16T23:50:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2406, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2406, "optime" : { "t" : 1358380234000, "i" : 8285 }, "optimeDate" : ISODate("2013-01-16T23:50:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:37Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2417, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2404, "optime" : { "t" : 1358380234000, "i" : 8285 }, "optimeDate" : ISODate("2013-01-16T23:50:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:37Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2404, "optime" : { "t" : 1358380234000, "i" : 8285 }, "optimeDate" : ISODate("2013-01-16T23:50:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2404, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2416, "optime" : { "t" : 1358380238000, "i" : 9447 }, "optimeDate" : ISODate("2013-01-16T23:50:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:50:39.819 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:50:40.449 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:50:42.450 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2422, "optime" : { "t" : 1358380243000, "i" : 1267 }, "optimeDate" : ISODate("2013-01-16T23:50:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2411, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2411, "optime" : { "t" : 1358380242000, "i" : 7174 }, "optimeDate" : ISODate("2013-01-16T23:50:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2421, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2408, "optime" : { "t" : 1358380242000, "i" : 8729 }, "optimeDate" : ISODate("2013-01-16T23:50:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2408, "optime" : { "t" : 1358380238000, "i" : 9447 }, "optimeDate" : ISODate("2013-01-16T23:50:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2408, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:43Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2420, "optime" : { "t" : 1358380243000, "i" : 1267 }, "optimeDate" : ISODate("2013-01-16T23:50:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:50:44.450 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:50:44.820 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:50:45.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 18:50:46.450 [conn144] end connection 127.0.0.2:54058 (2 connections now open) m31001| Wed Jan 16 18:50:46.450 [initandlisten] connection accepted from 127.0.0.2:36643 #145 (3 connections now open) m31000| Wed Jan 16 18:50:46.451 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2426, "optime" : { "t" : 1358380247000, "i" : 2754 }, "optimeDate" : ISODate("2013-01-16T23:50:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2415, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2415, "optime" : { "t" : 1358380243000, "i" : 1267 }, "optimeDate" : ISODate("2013-01-16T23:50:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:45Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2425, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2412, "optime" : { "t" : 1358380247000, "i" : 662 }, "optimeDate" : ISODate("2013-01-16T23:50:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2412, "optime" : { "t" : 1358380243000, "i" : 1267 }, "optimeDate" : ISODate("2013-01-16T23:50:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2412, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2424, "optime" : { "t" : 1358380247000, "i" : 2754 }, "optimeDate" : ISODate("2013-01-16T23:50:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:50:47.827 [conn145] end connection 127.0.0.2:44445 (3 connections now open) m31002| Wed Jan 16 18:50:48.327 [initandlisten] connection accepted from 127.0.0.2:44844 #147 (4 connections now open) m31000| Wed Jan 16 18:50:48.451 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:50:49.820 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:50:49.905 [conn90] end connection 127.0.0.4:37063 (4 connections now open) m31000| Wed Jan 16 18:50:49.905 [initandlisten] connection accepted from 127.0.0.4:34107 #91 (5 connections now open) m31000| Wed Jan 16 18:50:50.451 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2430, "optime" : { "t" : 1358380251000, "i" : 3905 }, "optimeDate" : ISODate("2013-01-16T23:50:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2419, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2419, "optime" : { "t" : 1358380250000, "i" : 4631 }, "optimeDate" : ISODate("2013-01-16T23:50:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2429, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2416, "optime" : { "t" : 1358380251000, "i" : 759 }, "optimeDate" : ISODate("2013-01-16T23:50:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2416, "optime" : { "t" : 1358380247000, "i" : 2754 }, "optimeDate" : ISODate("2013-01-16T23:50:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2416, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2428, "optime" : { "t" : 1358380251000, "i" : 3053 }, "optimeDate" : ISODate("2013-01-16T23:50:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:50:52.321 [conn143] end connection 127.0.0.4:44518 (2 connections now open) m31000| Wed Jan 16 18:50:52.452 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:50:52.821 [initandlisten] connection accepted from 127.0.0.4:43957 #146 (3 connections now open) m31000| Wed Jan 16 18:50:54.452 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:50:55.321 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2434, "optime" : { "t" : 1358380255000, "i" : 4388 }, "optimeDate" : ISODate("2013-01-16T23:50:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2423, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2423, "optime" : { "t" : 1358380251000, "i" : 3905 }, "optimeDate" : ISODate("2013-01-16T23:50:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:53Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2433, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2420, "optime" : { "t" : 1358380255000, "i" : 133 }, "optimeDate" : ISODate("2013-01-16T23:50:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:55Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2420, "optime" : { "t" : 1358380251000, "i" : 3905 }, "optimeDate" : ISODate("2013-01-16T23:50:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2420, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2432, "optime" : { "t" : 1358380255000, "i" : 4087 }, "optimeDate" : ISODate("2013-01-16T23:50:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:50:55.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:50:56.452 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:50:58.453 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2438, "optime" : { "t" : 1358380259000, "i" : 5595 }, "optimeDate" : ISODate("2013-01-16T23:50:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2427, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2427, "optime" : { "t" : 1358380255000, "i" : 4388 }, "optimeDate" : ISODate("2013-01-16T23:50:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2437, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2424, "optime" : { "t" : 1358380259000, "i" : 286 }, "optimeDate" : ISODate("2013-01-16T23:50:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:50:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2424, "optime" : { "t" : 1358380255000, "i" : 4388 }, "optimeDate" : ISODate("2013-01-16T23:50:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2424, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:50:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2436, "optime" : { "t" : 1358380259000, "i" : 5595 }, "optimeDate" : ISODate("2013-01-16T23:50:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:51:00.322 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:51:00.453 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:51:02.454 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:51:03.059 [conn146] end connection 127.0.0.3:34230 (3 connections now open) m31002| Wed Jan 16 18:51:03.059 [initandlisten] connection accepted from 127.0.0.3:39736 #148 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2442, "optime" : { "t" : 1358380263000, "i" : 7177 }, "optimeDate" : ISODate("2013-01-16T23:51:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2431, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2431, "optime" : { "t" : 1358380263000, "i" : 2862 }, "optimeDate" : ISODate("2013-01-16T23:51:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:50:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2441, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2428, "optime" : { "t" : 1358380263000, "i" : 536 }, "optimeDate" : ISODate("2013-01-16T23:51:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2428, "optime" : { "t" : 1358380259000, "i" : 5595 }, "optimeDate" : ISODate("2013-01-16T23:50:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:03Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2428, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:03Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2440, "optime" : { "t" : 1358380263000, "i" : 6753 }, "optimeDate" : ISODate("2013-01-16T23:51:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:51:04.454 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:51:05.323 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:51:06.454 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2446, "optime" : { "t" : 1358380267000, "i" : 8577 }, "optimeDate" : ISODate("2013-01-16T23:51:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2435, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2435, "optime" : { "t" : 1358380263000, "i" : 7177 }, "optimeDate" : ISODate("2013-01-16T23:51:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:05Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2445, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2432, "optime" : { "t" : 1358380267000, "i" : 1178 }, "optimeDate" : ISODate("2013-01-16T23:51:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:07Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2432, "optime" : { "t" : 1358380263000, "i" : 7177 }, "optimeDate" : ISODate("2013-01-16T23:51:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2432, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2444, "optime" : { "t" : 1358380267000, "i" : 8577 }, "optimeDate" : ISODate("2013-01-16T23:51:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:51:08.455 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:51:10.323 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:51:10.455 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2450, "optime" : { "t" : 1358380271000, "i" : 8745 }, "optimeDate" : ISODate("2013-01-16T23:51:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2439, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2439, "optime" : { "t" : 1358380267000, "i" : 8577 }, "optimeDate" : ISODate("2013-01-16T23:51:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:11Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2449, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2436, "optime" : { "t" : 1358380271000, "i" : 157 }, "optimeDate" : ISODate("2013-01-16T23:51:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2436, "optime" : { "t" : 1358380271000, "i" : 8392 }, "optimeDate" : ISODate("2013-01-16T23:51:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2436, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2448, "optime" : { "t" : 1358380271000, "i" : 8303 }, "optimeDate" : ISODate("2013-01-16T23:51:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:51:12.456 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:51:14.456 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:51:15.324 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2455, "optime" : { "t" : 1358380276000, "i" : 905 }, "optimeDate" : ISODate("2013-01-16T23:51:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2444, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2444, "optime" : { "t" : 1358380275000, "i" : 7075 }, "optimeDate" : ISODate("2013-01-16T23:51:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:15Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2454, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2441, "optime" : { "t" : 1358380275000, "i" : 781 }, "optimeDate" : ISODate("2013-01-16T23:51:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:15Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2441, "optime" : { "t" : 1358380275000, "i" : 8023 }, "optimeDate" : ISODate("2013-01-16T23:51:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2441, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2453, "optime" : { "t" : 1358380276000, "i" : 905 }, "optimeDate" : ISODate("2013-01-16T23:51:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:51:16.456 [conn145] end connection 127.0.0.2:36643 (2 connections now open) m31001| Wed Jan 16 18:51:16.456 [initandlisten] connection accepted from 127.0.0.2:55338 #147 (3 connections now open) m31000| Wed Jan 16 18:51:16.457 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:51:17.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:51:18.457 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:51:19.910 [conn91] end connection 127.0.0.4:34107 (4 connections now open) m31000| Wed Jan 16 18:51:19.910 [initandlisten] connection accepted from 127.0.0.4:37287 #92 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2459, "optime" : { "t" : 1358380280000, "i" : 2181 }, "optimeDate" : ISODate("2013-01-16T23:51:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2448, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2448, "optime" : { "t" : 1358380276000, "i" : 905 }, "optimeDate" : ISODate("2013-01-16T23:51:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:19Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2458, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2445, "optime" : { "t" : 1358380276000, "i" : 905 }, "optimeDate" : ISODate("2013-01-16T23:51:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2445, "optime" : { "t" : 1358380279000, "i" : 6782 }, "optimeDate" : ISODate("2013-01-16T23:51:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2445, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:19Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2457, "optime" : { "t" : 1358380280000, "i" : 1858 }, "optimeDate" : ISODate("2013-01-16T23:51:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:51:20.325 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:51:20.458 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:51:22.458 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2463, "optime" : { "t" : 1358380284000, "i" : 3371 }, "optimeDate" : ISODate("2013-01-16T23:51:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2452, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2452, "optime" : { "t" : 1358380283000, "i" : 252 }, "optimeDate" : ISODate("2013-01-16T23:51:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:23Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2462, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2449, "optime" : { "t" : 1358380280000, "i" : 2181 }, "optimeDate" : ISODate("2013-01-16T23:51:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2449, "optime" : { "t" : 1358380283000, "i" : 5677 }, "optimeDate" : ISODate("2013-01-16T23:51:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2449, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2461, "optime" : { "t" : 1358380284000, "i" : 3371 }, "optimeDate" : ISODate("2013-01-16T23:51:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:51:24.458 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:51:25.325 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:51:25.832 [conn147] end connection 127.0.0.2:44844 (3 connections now open) m31002| Wed Jan 16 18:51:26.332 [initandlisten] connection accepted from 127.0.0.2:55416 #149 (4 connections now open) m31000| Wed Jan 16 18:51:26.459 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:51:27.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:51:28.459 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2467, "optime" : { "t" : 1358380288000, "i" : 4667 }, "optimeDate" : ISODate("2013-01-16T23:51:28Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2456, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2456, "optime" : { "t" : 1358380284000, "i" : 3371 }, "optimeDate" : ISODate("2013-01-16T23:51:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:27Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2466, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2453, "optime" : { "t" : 1358380284000, "i" : 3371 }, "optimeDate" : ISODate("2013-01-16T23:51:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:27Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:28Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2453, "optime" : { "t" : 1358380287000, "i" : 4365 }, "optimeDate" : ISODate("2013-01-16T23:51:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2453, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2465, "optime" : { "t" : 1358380288000, "i" : 4667 }, "optimeDate" : ISODate("2013-01-16T23:51:28Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:51:30.326 [conn146] end connection 127.0.0.4:43957 (2 connections now open) m31000| Wed Jan 16 18:51:30.459 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:51:30.826 [initandlisten] connection accepted from 127.0.0.4:34137 #148 (3 connections now open) m31002| Wed Jan 16 18:51:30.826 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:51:32.460 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2471, "optime" : { "t" : 1358380292000, "i" : 6045 }, "optimeDate" : ISODate("2013-01-16T23:51:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2460, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2460, "optime" : { "t" : 1358380288000, "i" : 4667 }, "optimeDate" : ISODate("2013-01-16T23:51:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:31Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2470, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2457, "optime" : { "t" : 1358380288000, "i" : 4667 }, "optimeDate" : ISODate("2013-01-16T23:51:28Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2457, "optime" : { "t" : 1358380291000, "i" : 3223 }, "optimeDate" : ISODate("2013-01-16T23:51:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2457, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:31Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2469, "optime" : { "t" : 1358380292000, "i" : 6045 }, "optimeDate" : ISODate("2013-01-16T23:51:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:51:33.064 [conn148] end connection 127.0.0.3:39736 (3 connections now open) m31002| Wed Jan 16 18:51:33.064 [initandlisten] connection accepted from 127.0.0.3:43115 #150 (4 connections now open) m31000| Wed Jan 16 18:51:34.460 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:51:35.827 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:51:36.460 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2475, "optime" : { "t" : 1358380296000, "i" : 7560 }, "optimeDate" : ISODate("2013-01-16T23:51:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2464, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2464, "optime" : { "t" : 1358380296000, "i" : 2827 }, "optimeDate" : ISODate("2013-01-16T23:51:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2474, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2461, "optime" : { "t" : 1358380292000, "i" : 6045 }, "optimeDate" : ISODate("2013-01-16T23:51:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:35Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2461, "optime" : { "t" : 1358380295000, "i" : 1784 }, "optimeDate" : ISODate("2013-01-16T23:51:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2461, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2473, "optime" : { "t" : 1358380296000, "i" : 6891 }, "optimeDate" : ISODate("2013-01-16T23:51:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:51:38.461 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:51:40.461 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:51:40.828 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2480, "optime" : { "t" : 1358380301000, "i" : 876 }, "optimeDate" : ISODate("2013-01-16T23:51:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2469, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2469, "optime" : { "t" : 1358380296000, "i" : 7560 }, "optimeDate" : ISODate("2013-01-16T23:51:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:39Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2479, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2466, "optime" : { "t" : 1358380301000, "i" : 714 }, "optimeDate" : ISODate("2013-01-16T23:51:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:41Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2466, "optime" : { "t" : 1358380299000, "i" : 143 }, "optimeDate" : ISODate("2013-01-16T23:51:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2466, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:41Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2478, "optime" : { "t" : 1358380301000, "i" : 714 }, "optimeDate" : ISODate("2013-01-16T23:51:41Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:51:42.461 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:51:44.462 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2484, "optime" : { "t" : 1358380305000, "i" : 2336 }, "optimeDate" : ISODate("2013-01-16T23:51:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2473, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2473, "optime" : { "t" : 1358380301000, "i" : 876 }, "optimeDate" : ISODate("2013-01-16T23:51:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:43Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2483, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2470, "optime" : { "t" : 1358380305000, "i" : 748 }, "optimeDate" : ISODate("2013-01-16T23:51:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2470, "optime" : { "t" : 1358380301000, "i" : 876 }, "optimeDate" : ISODate("2013-01-16T23:51:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2470, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2482, "optime" : { "t" : 1358380305000, "i" : 2074 }, "optimeDate" : ISODate("2013-01-16T23:51:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:51:45.829 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:51:46.462 [conn147] end connection 127.0.0.2:55338 (2 connections now open) m31001| Wed Jan 16 18:51:46.462 [initandlisten] connection accepted from 127.0.0.2:46033 #149 (3 connections now open) m31000| Wed Jan 16 18:51:46.462 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:51:48.463 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:51:48.709 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:477721 238ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2488, "optime" : { "t" : 1358380309000, "i" : 2701 }, "optimeDate" : ISODate("2013-01-16T23:51:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2477, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2477, "optime" : { "t" : 1358380308000, "i" : 1662 }, "optimeDate" : ISODate("2013-01-16T23:51:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2487, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2474, "optime" : { "t" : 1358380309000, "i" : 386 }, "optimeDate" : ISODate("2013-01-16T23:51:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2474, "optime" : { "t" : 1358380305000, "i" : 2336 }, "optimeDate" : ISODate("2013-01-16T23:51:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2474, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2486, "optime" : { "t" : 1358380309000, "i" : 2450 }, "optimeDate" : ISODate("2013-01-16T23:51:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:51:49.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:51:49.915 [conn92] end connection 127.0.0.4:37287 (4 connections now open) m31000| Wed Jan 16 18:51:49.915 [initandlisten] connection accepted from 127.0.0.4:41374 #93 (5 connections now open) m31000| Wed Jan 16 18:51:50.463 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:51:50.829 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:51:52.463 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:51:53.594 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:517899 258ms { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2492, "optime" : { "t" : 1358380313000, "i" : 3485 }, "optimeDate" : ISODate("2013-01-16T23:51:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2481, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2481, "optime" : { "t" : 1358380309000, "i" : 2701 }, "optimeDate" : ISODate("2013-01-16T23:51:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:51Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2491, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2478, "optime" : { "t" : 1358380313000, "i" : 300 }, "optimeDate" : ISODate("2013-01-16T23:51:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:53Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2478, "optime" : { "t" : 1358380309000, "i" : 2701 }, "optimeDate" : ISODate("2013-01-16T23:51:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2478, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2490, "optime" : { "t" : 1358380313000, "i" : 3432 }, "optimeDate" : ISODate("2013-01-16T23:51:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:51:54.464 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:51:55.830 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:51:56.464 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2496, "optime" : { "t" : 1358380317000, "i" : 6564 }, "optimeDate" : ISODate("2013-01-16T23:51:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2485, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2485, "optime" : { "t" : 1358380313000, "i" : 3485 }, "optimeDate" : ISODate("2013-01-16T23:51:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2495, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2482, "optime" : { "t" : 1358380317000, "i" : 567 }, "optimeDate" : ISODate("2013-01-16T23:51:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:51:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2482, "optime" : { "t" : 1358380313000, "i" : 3485 }, "optimeDate" : ISODate("2013-01-16T23:51:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2482, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:51:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2494, "optime" : { "t" : 1358380317000, "i" : 6564 }, "optimeDate" : ISODate("2013-01-16T23:51:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:51:58.465 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:51:59.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:52:00.465 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:52:00.830 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2500, "optime" : { "t" : 1358380321000, "i" : 8588 }, "optimeDate" : ISODate("2013-01-16T23:52:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2489, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2489, "optime" : { "t" : 1358380321000, "i" : 2431 }, "optimeDate" : ISODate("2013-01-16T23:52:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:01Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2499, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2486, "optime" : { "t" : 1358380321000, "i" : 588 }, "optimeDate" : ISODate("2013-01-16T23:52:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2486, "optime" : { "t" : 1358380321000, "i" : 7994 }, "optimeDate" : ISODate("2013-01-16T23:52:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2486, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2498, "optime" : { "t" : 1358380321000, "i" : 8588 }, "optimeDate" : ISODate("2013-01-16T23:52:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:52:02.465 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:52:03.069 [conn150] end connection 127.0.0.3:43115 (3 connections now open) m31002| Wed Jan 16 18:52:03.070 [initandlisten] connection accepted from 127.0.0.3:42619 #151 (4 connections now open) m31002| Wed Jan 16 18:52:03.837 [conn149] end connection 127.0.0.2:55416 (3 connections now open) m31002| Wed Jan 16 18:52:04.337 [initandlisten] connection accepted from 127.0.0.2:34165 #152 (4 connections now open) m31000| Wed Jan 16 18:52:04.466 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:52:05.831 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2505, "optime" : { "t" : 1358380326000, "i" : 818 }, "optimeDate" : ISODate("2013-01-16T23:52:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2494, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2494, "optime" : { "t" : 1358380321000, "i" : 8588 }, "optimeDate" : ISODate("2013-01-16T23:52:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:05Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:51:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2504, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2491, "optime" : { "t" : 1358380325000, "i" : 153 }, "optimeDate" : ISODate("2013-01-16T23:52:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:05Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2491, "optime" : { "t" : 1358380325000, "i" : 8106 }, "optimeDate" : ISODate("2013-01-16T23:52:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2491, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2503, "optime" : { "t" : 1358380326000, "i" : 741 }, "optimeDate" : ISODate("2013-01-16T23:52:06Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:52:06.466 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:52:08.331 [conn148] end connection 127.0.0.4:34137 (2 connections now open) m31000| Wed Jan 16 18:52:08.467 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:52:08.831 [initandlisten] connection accepted from 127.0.0.4:43816 #150 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2509, "optime" : { "t" : 1358380330000, "i" : 2333 }, "optimeDate" : ISODate("2013-01-16T23:52:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2498, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2498, "optime" : { "t" : 1358380329000, "i" : 1378 }, "optimeDate" : ISODate("2013-01-16T23:52:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:09Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2508, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2495, "optime" : { "t" : 1358380326000, "i" : 818 }, "optimeDate" : ISODate("2013-01-16T23:52:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2495, "optime" : { "t" : 1358380329000, "i" : 7323 }, "optimeDate" : ISODate("2013-01-16T23:52:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2495, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:09Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2507, "optime" : { "t" : 1358380330000, "i" : 2333 }, "optimeDate" : ISODate("2013-01-16T23:52:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:52:10.467 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:52:11.332 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:52:12.467 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2513, "optime" : { "t" : 1358380334000, "i" : 2935 }, "optimeDate" : ISODate("2013-01-16T23:52:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2502, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2502, "optime" : { "t" : 1358380334000, "i" : 2643 }, "optimeDate" : ISODate("2013-01-16T23:52:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2512, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2499, "optime" : { "t" : 1358380330000, "i" : 2333 }, "optimeDate" : ISODate("2013-01-16T23:52:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:13Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2499, "optime" : { "t" : 1358380333000, "i" : 6427 }, "optimeDate" : ISODate("2013-01-16T23:52:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2499, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2511, "optime" : { "t" : 1358380334000, "i" : 2935 }, "optimeDate" : ISODate("2013-01-16T23:52:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:52:14.468 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:52:16.333 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:52:16.468 [conn149] end connection 127.0.0.2:46033 (2 connections now open) m31001| Wed Jan 16 18:52:16.468 [initandlisten] connection accepted from 127.0.0.2:34627 #151 (3 connections now open) m31000| Wed Jan 16 18:52:16.469 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:52:18.469 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2517, "optime" : { "t" : 1358380338000, "i" : 4386 }, "optimeDate" : ISODate("2013-01-16T23:52:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2506, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2506, "optime" : { "t" : 1358380334000, "i" : 2935 }, "optimeDate" : ISODate("2013-01-16T23:52:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:17Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2516, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2503, "optime" : { "t" : 1358380334000, "i" : 2935 }, "optimeDate" : ISODate("2013-01-16T23:52:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2503, "optime" : { "t" : 1358380337000, "i" : 4530 }, "optimeDate" : ISODate("2013-01-16T23:52:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2503, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2515, "optime" : { "t" : 1358380338000, "i" : 4363 }, "optimeDate" : ISODate("2013-01-16T23:52:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:52:19.921 [conn93] end connection 127.0.0.4:41374 (4 connections now open) m31000| Wed Jan 16 18:52:19.921 [initandlisten] connection accepted from 127.0.0.4:49202 #94 (5 connections now open) m31000| Wed Jan 16 18:52:20.469 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:52:21.334 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:52:21.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:52:22.470 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2521, "optime" : { "t" : 1358380342000, "i" : 5875 }, "optimeDate" : ISODate("2013-01-16T23:52:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2510, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2510, "optime" : { "t" : 1358380341000, "i" : 2690 }, "optimeDate" : ISODate("2013-01-16T23:52:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:21Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2520, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2507, "optime" : { "t" : 1358380338000, "i" : 4386 }, "optimeDate" : ISODate("2013-01-16T23:52:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:21Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2507, "optime" : { "t" : 1358380341000, "i" : 2982 }, "optimeDate" : ISODate("2013-01-16T23:52:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2507, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2519, "optime" : { "t" : 1358380342000, "i" : 5875 }, "optimeDate" : ISODate("2013-01-16T23:52:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:52:24.470 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:52:26.334 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:52:26.470 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2525, "optime" : { "t" : 1358380346000, "i" : 7788 }, "optimeDate" : ISODate("2013-01-16T23:52:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2514, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2514, "optime" : { "t" : 1358380346000, "i" : 7214 }, "optimeDate" : ISODate("2013-01-16T23:52:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2524, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2511, "optime" : { "t" : 1358380342000, "i" : 5875 }, "optimeDate" : ISODate("2013-01-16T23:52:22Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:26Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2511, "optime" : { "t" : 1358380345000, "i" : 1516 }, "optimeDate" : ISODate("2013-01-16T23:52:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2511, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2523, "optime" : { "t" : 1358380346000, "i" : 7214 }, "optimeDate" : ISODate("2013-01-16T23:52:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:52:28.471 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:52:30.471 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2530, "optime" : { "t" : 1358380351000, "i" : 325 }, "optimeDate" : ISODate("2013-01-16T23:52:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2519, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2519, "optime" : { "t" : 1358380346000, "i" : 7788 }, "optimeDate" : ISODate("2013-01-16T23:52:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:29Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2529, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2516, "optime" : { "t" : 1358380346000, "i" : 7788 }, "optimeDate" : ISODate("2013-01-16T23:52:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2516, "optime" : { "t" : 1358380349000, "i" : 307 }, "optimeDate" : ISODate("2013-01-16T23:52:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2516, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:29Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2528, "optime" : { "t" : 1358380351000, "i" : 279 }, "optimeDate" : ISODate("2013-01-16T23:52:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:52:31.335 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:52:31.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:52:32.471 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:52:33.075 [conn151] end connection 127.0.0.3:42619 (3 connections now open) m31002| Wed Jan 16 18:52:33.075 [initandlisten] connection accepted from 127.0.0.3:52271 #153 (4 connections now open) m31000| Wed Jan 16 18:52:34.472 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2534, "optime" : { "t" : 1358380355000, "i" : 1623 }, "optimeDate" : ISODate("2013-01-16T23:52:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2523, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2523, "optime" : { "t" : 1358380354000, "i" : 2237 }, "optimeDate" : ISODate("2013-01-16T23:52:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2533, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2520, "optime" : { "t" : 1358380355000, "i" : 391 }, "optimeDate" : ISODate("2013-01-16T23:52:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2520, "optime" : { "t" : 1358380351000, "i" : 325 }, "optimeDate" : ISODate("2013-01-16T23:52:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2520, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:35Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2532, "optime" : { "t" : 1358380355000, "i" : 1623 }, "optimeDate" : ISODate("2013-01-16T23:52:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:52:36.336 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:52:36.472 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:52:38.472 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2538, "optime" : { "t" : 1358380359000, "i" : 3118 }, "optimeDate" : ISODate("2013-01-16T23:52:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2527, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2527, "optime" : { "t" : 1358380359000, "i" : 2762 }, "optimeDate" : ISODate("2013-01-16T23:52:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2537, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2524, "optime" : { "t" : 1358380359000, "i" : 228 }, "optimeDate" : ISODate("2013-01-16T23:52:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2524, "optime" : { "t" : 1358380355000, "i" : 1623 }, "optimeDate" : ISODate("2013-01-16T23:52:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:39Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2524, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:39Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2536, "optime" : { "t" : 1358380359000, "i" : 3118 }, "optimeDate" : ISODate("2013-01-16T23:52:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:52:40.473 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:52:41.336 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:52:41.843 [conn152] end connection 127.0.0.2:34165 (3 connections now open) m31002| Wed Jan 16 18:52:42.343 [initandlisten] connection accepted from 127.0.0.2:38297 #154 (4 connections now open) m31000| Wed Jan 16 18:52:42.473 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2542, "optime" : { "t" : 1358380363000, "i" : 4605 }, "optimeDate" : ISODate("2013-01-16T23:52:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2531, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2531, "optime" : { "t" : 1358380359000, "i" : 3118 }, "optimeDate" : ISODate("2013-01-16T23:52:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2541, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2528, "optime" : { "t" : 1358380363000, "i" : 287 }, "optimeDate" : ISODate("2013-01-16T23:52:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2528, "optime" : { "t" : 1358380359000, "i" : 3118 }, "optimeDate" : ISODate("2013-01-16T23:52:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2528, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:43Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2540, "optime" : { "t" : 1358380363000, "i" : 4194 }, "optimeDate" : ISODate("2013-01-16T23:52:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:52:44.473 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:52:46.337 [conn150] end connection 127.0.0.4:43816 (2 connections now open) m31001| Wed Jan 16 18:52:46.473 [conn151] end connection 127.0.0.2:34627 (1 connection now open) m31001| Wed Jan 16 18:52:46.474 [initandlisten] connection accepted from 127.0.0.2:56014 #152 (2 connections now open) m31000| Wed Jan 16 18:52:46.474 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:52:46.837 [initandlisten] connection accepted from 127.0.0.4:37508 #153 (3 connections now open) m31002| Wed Jan 16 18:52:46.837 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2546, "optime" : { "t" : 1358380367000, "i" : 5755 }, "optimeDate" : ISODate("2013-01-16T23:52:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2535, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2535, "optime" : { "t" : 1358380367000, "i" : 2934 }, "optimeDate" : ISODate("2013-01-16T23:52:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2545, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2532, "optime" : { "t" : 1358380367000, "i" : 198 }, "optimeDate" : ISODate("2013-01-16T23:52:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2532, "optime" : { "t" : 1358380363000, "i" : 4605 }, "optimeDate" : ISODate("2013-01-16T23:52:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:47Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2532, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2544, "optime" : { "t" : 1358380367000, "i" : 5598 }, "optimeDate" : ISODate("2013-01-16T23:52:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:52:48.474 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:52:49.926 [conn94] end connection 127.0.0.4:49202 (4 connections now open) m31000| Wed Jan 16 18:52:49.926 [initandlisten] connection accepted from 127.0.0.4:51681 #95 (5 connections now open) m31000| Wed Jan 16 18:52:50.475 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:52:51.717 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:310405 155ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2550, "optime" : { "t" : 1358380371000, "i" : 7408 }, "optimeDate" : ISODate("2013-01-16T23:52:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2539, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2539, "optime" : { "t" : 1358380367000, "i" : 5755 }, "optimeDate" : ISODate("2013-01-16T23:52:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:49Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2549, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2536, "optime" : { "t" : 1358380371000, "i" : 700 }, "optimeDate" : ISODate("2013-01-16T23:52:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2536, "optime" : { "t" : 1358380367000, "i" : 5755 }, "optimeDate" : ISODate("2013-01-16T23:52:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2536, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2548, "optime" : { "t" : 1358380371000, "i" : 7408 }, "optimeDate" : ISODate("2013-01-16T23:52:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:52:51.838 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:52:52.475 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:52:53.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:52:54.476 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2554, "optime" : { "t" : 1358380375000, "i" : 8364 }, "optimeDate" : ISODate("2013-01-16T23:52:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2543, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2543, "optime" : { "t" : 1358380374000, "i" : 282 }, "optimeDate" : ISODate("2013-01-16T23:52:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2553, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2540, "optime" : { "t" : 1358380375000, "i" : 425 }, "optimeDate" : ISODate("2013-01-16T23:52:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:52:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2540, "optime" : { "t" : 1358380371000, "i" : 7408 }, "optimeDate" : ISODate("2013-01-16T23:52:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2540, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2552, "optime" : { "t" : 1358380375000, "i" : 8364 }, "optimeDate" : ISODate("2013-01-16T23:52:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:52:56.476 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:52:56.839 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:52:58.476 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2559, "optime" : { "t" : 1358380380000, "i" : 612 }, "optimeDate" : ISODate("2013-01-16T23:53:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2548, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2548, "optime" : { "t" : 1358380379000, "i" : 7249 }, "optimeDate" : ISODate("2013-01-16T23:52:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:59Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2558, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2545, "optime" : { "t" : 1358380379000, "i" : 568 }, "optimeDate" : ISODate("2013-01-16T23:52:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:52:59Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2545, "optime" : { "t" : 1358380379000, "i" : 8153 }, "optimeDate" : ISODate("2013-01-16T23:52:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2545, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2557, "optime" : { "t" : 1358380380000, "i" : 612 }, "optimeDate" : ISODate("2013-01-16T23:53:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:53:00.477 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:53:01.839 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:53:02.477 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:53:03.080 [conn153] end connection 127.0.0.3:52271 (3 connections now open) m31002| Wed Jan 16 18:53:03.080 [initandlisten] connection accepted from 127.0.0.3:52784 #155 (4 connections now open) m31001| Wed Jan 16 18:53:03.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:53:03.718 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:260624 130ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2563, "optime" : { "t" : 1358380384000, "i" : 1858 }, "optimeDate" : ISODate("2013-01-16T23:53:04Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2552, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2552, "optime" : { "t" : 1358380380000, "i" : 612 }, "optimeDate" : ISODate("2013-01-16T23:53:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:03Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2562, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2549, "optime" : { "t" : 1358380383000, "i" : 147 }, "optimeDate" : ISODate("2013-01-16T23:53:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2549, "optime" : { "t" : 1358380383000, "i" : 7644 }, "optimeDate" : ISODate("2013-01-16T23:53:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2549, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:03Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2561, "optime" : { "t" : 1358380384000, "i" : 1858 }, "optimeDate" : ISODate("2013-01-16T23:53:04Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:53:04.478 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:53:06.478 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:53:06.840 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2567, "optime" : { "t" : 1358380388000, "i" : 3618 }, "optimeDate" : ISODate("2013-01-16T23:53:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2556, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2556, "optime" : { "t" : 1358380387000, "i" : 784 }, "optimeDate" : ISODate("2013-01-16T23:53:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:07Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2566, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2553, "optime" : { "t" : 1358380384000, "i" : 1858 }, "optimeDate" : ISODate("2013-01-16T23:53:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2553, "optime" : { "t" : 1358380387000, "i" : 6113 }, "optimeDate" : ISODate("2013-01-16T23:53:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2553, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:07Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2565, "optime" : { "t" : 1358380388000, "i" : 3618 }, "optimeDate" : ISODate("2013-01-16T23:53:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:53:08.478 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:53:10.479 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:53:11.841 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:53:12.479 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2571, "optime" : { "t" : 1358380392000, "i" : 4089 }, "optimeDate" : ISODate("2013-01-16T23:53:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2560, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2560, "optime" : { "t" : 1358380392000, "i" : 2506 }, "optimeDate" : ISODate("2013-01-16T23:53:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:52:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2570, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2557, "optime" : { "t" : 1358380388000, "i" : 3618 }, "optimeDate" : ISODate("2013-01-16T23:53:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:11Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2557, "optime" : { "t" : 1358380391000, "i" : 5182 }, "optimeDate" : ISODate("2013-01-16T23:53:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2557, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2569, "optime" : { "t" : 1358380392000, "i" : 3810 }, "optimeDate" : ISODate("2013-01-16T23:53:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:53:14.479 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:53:16.480 [conn152] end connection 127.0.0.2:56014 (2 connections now open) m31001| Wed Jan 16 18:53:16.480 [initandlisten] connection accepted from 127.0.0.2:33406 #154 (3 connections now open) m31000| Wed Jan 16 18:53:16.480 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2575, "optime" : { "t" : 1358380396000, "i" : 5547 }, "optimeDate" : ISODate("2013-01-16T23:53:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2564, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2564, "optime" : { "t" : 1358380392000, "i" : 4089 }, "optimeDate" : ISODate("2013-01-16T23:53:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:15Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2574, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2561, "optime" : { "t" : 1358380392000, "i" : 4089 }, "optimeDate" : ISODate("2013-01-16T23:53:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2561, "optime" : { "t" : 1358380395000, "i" : 4013 }, "optimeDate" : ISODate("2013-01-16T23:53:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2561, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:15Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2573, "optime" : { "t" : 1358380396000, "i" : 5456 }, "optimeDate" : ISODate("2013-01-16T23:53:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:53:16.841 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:53:18.480 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:53:19.848 [conn154] end connection 127.0.0.2:38297 (3 connections now open) m31000| Wed Jan 16 18:53:19.931 [conn95] end connection 127.0.0.4:51681 (4 connections now open) m31000| Wed Jan 16 18:53:19.931 [initandlisten] connection accepted from 127.0.0.4:60566 #96 (5 connections now open) m31002| Wed Jan 16 18:53:20.348 [initandlisten] connection accepted from 127.0.0.2:47214 #156 (4 connections now open) m31000| Wed Jan 16 18:53:20.481 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2579, "optime" : { "t" : 1358380400000, "i" : 7388 }, "optimeDate" : ISODate("2013-01-16T23:53:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2568, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2568, "optime" : { "t" : 1358380400000, "i" : 2964 }, "optimeDate" : ISODate("2013-01-16T23:53:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2578, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2565, "optime" : { "t" : 1358380396000, "i" : 5547 }, "optimeDate" : ISODate("2013-01-16T23:53:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:19Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2565, "optime" : { "t" : 1358380399000, "i" : 1735 }, "optimeDate" : ISODate("2013-01-16T23:53:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2565, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2577, "optime" : { "t" : 1358380400000, "i" : 7388 }, "optimeDate" : ISODate("2013-01-16T23:53:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:53:21.842 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:53:22.481 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:53:24.342 [conn153] end connection 127.0.0.4:37508 (2 connections now open) m31000| Wed Jan 16 18:53:24.481 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:53:24.842 [initandlisten] connection accepted from 127.0.0.4:44332 #155 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2583, "optime" : { "t" : 1358380404000, "i" : 8805 }, "optimeDate" : ISODate("2013-01-16T23:53:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2572, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2572, "optime" : { "t" : 1358380400000, "i" : 7388 }, "optimeDate" : ISODate("2013-01-16T23:53:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:23Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2582, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2569, "optime" : { "t" : 1358380400000, "i" : 7388 }, "optimeDate" : ISODate("2013-01-16T23:53:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:24Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2569, "optime" : { "t" : 1358380403000, "i" : 481 }, "optimeDate" : ISODate("2013-01-16T23:53:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2569, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2581, "optime" : { "t" : 1358380404000, "i" : 8805 }, "optimeDate" : ISODate("2013-01-16T23:53:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:53:25.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:53:26.482 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:53:27.343 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:53:28.482 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:53:28.645 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:286297 143ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2588, "optime" : { "t" : 1358380409000, "i" : 772 }, "optimeDate" : ISODate("2013-01-16T23:53:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2577, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2577, "optime" : { "t" : 1358380404000, "i" : 8805 }, "optimeDate" : ISODate("2013-01-16T23:53:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:27Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2587, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2574, "optime" : { "t" : 1358380409000, "i" : 527 }, "optimeDate" : ISODate("2013-01-16T23:53:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2574, "optime" : { "t" : 1358380404000, "i" : 8805 }, "optimeDate" : ISODate("2013-01-16T23:53:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2574, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:29Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2586, "optime" : { "t" : 1358380409000, "i" : 528 }, "optimeDate" : ISODate("2013-01-16T23:53:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:53:30.482 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:53:32.344 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:53:32.483 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:53:33.086 [conn155] end connection 127.0.0.3:52784 (3 connections now open) m31002| Wed Jan 16 18:53:33.086 [initandlisten] connection accepted from 127.0.0.3:52366 #157 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2592, "optime" : { "t" : 1358380413000, "i" : 2816 }, "optimeDate" : ISODate("2013-01-16T23:53:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2581, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2581, "optime" : { "t" : 1358380412000, "i" : 5918 }, "optimeDate" : ISODate("2013-01-16T23:53:32Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2591, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2578, "optime" : { "t" : 1358380413000, "i" : 667 }, "optimeDate" : ISODate("2013-01-16T23:53:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2578, "optime" : { "t" : 1358380409000, "i" : 772 }, "optimeDate" : ISODate("2013-01-16T23:53:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2578, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:33Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2590, "optime" : { "t" : 1358380413000, "i" : 2816 }, "optimeDate" : ISODate("2013-01-16T23:53:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:53:34.483 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:53:35.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:53:36.483 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:53:37.344 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2596, "optime" : { "t" : 1358380417000, "i" : 4304 }, "optimeDate" : ISODate("2013-01-16T23:53:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2585, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2585, "optime" : { "t" : 1358380413000, "i" : 2816 }, "optimeDate" : ISODate("2013-01-16T23:53:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:35Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2595, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2582, "optime" : { "t" : 1358380417000, "i" : 1001 }, "optimeDate" : ISODate("2013-01-16T23:53:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:37Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2582, "optime" : { "t" : 1358380413000, "i" : 2816 }, "optimeDate" : ISODate("2013-01-16T23:53:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2582, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2594, "optime" : { "t" : 1358380417000, "i" : 3923 }, "optimeDate" : ISODate("2013-01-16T23:53:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:53:38.484 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:53:40.484 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2600, "optime" : { "t" : 1358380421000, "i" : 5888 }, "optimeDate" : ISODate("2013-01-16T23:53:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2589, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2589, "optime" : { "t" : 1358380417000, "i" : 4304 }, "optimeDate" : ISODate("2013-01-16T23:53:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2599, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2586, "optime" : { "t" : 1358380421000, "i" : 198 }, "optimeDate" : ISODate("2013-01-16T23:53:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:41Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2586, "optime" : { "t" : 1358380417000, "i" : 4304 }, "optimeDate" : ISODate("2013-01-16T23:53:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2586, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:41Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2598, "optime" : { "t" : 1358380421000, "i" : 5888 }, "optimeDate" : ISODate("2013-01-16T23:53:41Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:53:42.345 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:53:42.485 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:53:44.485 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2604, "optime" : { "t" : 1358380425000, "i" : 8158 }, "optimeDate" : ISODate("2013-01-16T23:53:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2593, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2593, "optime" : { "t" : 1358380425000, "i" : 3085 }, "optimeDate" : ISODate("2013-01-16T23:53:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2603, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2590, "optime" : { "t" : 1358380425000, "i" : 691 }, "optimeDate" : ISODate("2013-01-16T23:53:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2590, "optime" : { "t" : 1358380421000, "i" : 5888 }, "optimeDate" : ISODate("2013-01-16T23:53:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:45Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2590, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2602, "optime" : { "t" : 1358380425000, "i" : 8158 }, "optimeDate" : ISODate("2013-01-16T23:53:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:53:46.485 [conn154] end connection 127.0.0.2:33406 (2 connections now open) m31001| Wed Jan 16 18:53:46.486 [initandlisten] connection accepted from 127.0.0.2:55618 #156 (3 connections now open) m31000| Wed Jan 16 18:53:46.486 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:53:47.346 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:53:48.486 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:53:49.936 [conn96] end connection 127.0.0.4:60566 (4 connections now open) m31000| Wed Jan 16 18:53:49.937 [initandlisten] connection accepted from 127.0.0.4:54350 #97 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2609, "optime" : { "t" : 1358380430000, "i" : 695 }, "optimeDate" : ISODate("2013-01-16T23:53:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2598, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2598, "optime" : { "t" : 1358380425000, "i" : 8158 }, "optimeDate" : ISODate("2013-01-16T23:53:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:49Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2608, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2595, "optime" : { "t" : 1358380428000, "i" : 555 }, "optimeDate" : ISODate("2013-01-16T23:53:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:49Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2595, "optime" : { "t" : 1358380429000, "i" : 8647 }, "optimeDate" : ISODate("2013-01-16T23:53:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2595, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2607, "optime" : { "t" : 1358380430000, "i" : 359 }, "optimeDate" : ISODate("2013-01-16T23:53:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:53:50.487 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:53:52.346 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:53:52.487 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:53:53.215 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:304640 152ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2613, "optime" : { "t" : 1358380434000, "i" : 1363 }, "optimeDate" : ISODate("2013-01-16T23:53:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2602, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2602, "optime" : { "t" : 1358380430000, "i" : 695 }, "optimeDate" : ISODate("2013-01-16T23:53:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:53Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2612, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2599, "optime" : { "t" : 1358380433000, "i" : 30 }, "optimeDate" : ISODate("2013-01-16T23:53:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2599, "optime" : { "t" : 1358380433000, "i" : 8394 }, "optimeDate" : ISODate("2013-01-16T23:53:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2599, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:53Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2611, "optime" : { "t" : 1358380434000, "i" : 1363 }, "optimeDate" : ISODate("2013-01-16T23:53:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:53:54.488 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:53:56.488 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:53:57.347 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:53:57.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:53:57.854 [conn156] end connection 127.0.0.2:47214 (3 connections now open) m31002| Wed Jan 16 18:53:58.354 [initandlisten] connection accepted from 127.0.0.2:60775 #158 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2617, "optime" : { "t" : 1358380438000, "i" : 3143 }, "optimeDate" : ISODate("2013-01-16T23:53:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2606, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2606, "optime" : { "t" : 1358380438000, "i" : 2769 }, "optimeDate" : ISODate("2013-01-16T23:53:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2616, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2603, "optime" : { "t" : 1358380434000, "i" : 1363 }, "optimeDate" : ISODate("2013-01-16T23:53:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:57Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:53:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2603, "optime" : { "t" : 1358380437000, "i" : 5786 }, "optimeDate" : ISODate("2013-01-16T23:53:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:53:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2603, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2615, "optime" : { "t" : 1358380438000, "i" : 3143 }, "optimeDate" : ISODate("2013-01-16T23:53:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:53:58.488 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:54:00.489 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:54:02.347 [conn155] end connection 127.0.0.4:44332 (2 connections now open) m31000| Wed Jan 16 18:54:02.489 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2621, "optime" : { "t" : 1358380442000, "i" : 4563 }, "optimeDate" : ISODate("2013-01-16T23:54:02Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2610, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2610, "optime" : { "t" : 1358380438000, "i" : 3143 }, "optimeDate" : ISODate("2013-01-16T23:53:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:01Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2620, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2607, "optime" : { "t" : 1358380438000, "i" : 3143 }, "optimeDate" : ISODate("2013-01-16T23:53:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2607, "optime" : { "t" : 1358380441000, "i" : 4591 }, "optimeDate" : ISODate("2013-01-16T23:54:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2607, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2619, "optime" : { "t" : 1358380442000, "i" : 4375 }, "optimeDate" : ISODate("2013-01-16T23:54:02Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:54:02.847 [initandlisten] connection accepted from 127.0.0.4:53013 #157 (3 connections now open) m31002| Wed Jan 16 18:54:02.848 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:54:03.091 [conn157] end connection 127.0.0.3:52366 (3 connections now open) m31002| Wed Jan 16 18:54:03.091 [initandlisten] connection accepted from 127.0.0.3:42174 #159 (4 connections now open) m31000| Wed Jan 16 18:54:04.489 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:54:06.490 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2625, "optime" : { "t" : 1358380446000, "i" : 6078 }, "optimeDate" : ISODate("2013-01-16T23:54:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2614, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2614, "optime" : { "t" : 1358380445000, "i" : 2531 }, "optimeDate" : ISODate("2013-01-16T23:54:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:05Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2624, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2611, "optime" : { "t" : 1358380442000, "i" : 4563 }, "optimeDate" : ISODate("2013-01-16T23:54:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:05Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2611, "optime" : { "t" : 1358380445000, "i" : 3023 }, "optimeDate" : ISODate("2013-01-16T23:54:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2611, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2623, "optime" : { "t" : 1358380446000, "i" : 6078 }, "optimeDate" : ISODate("2013-01-16T23:54:06Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:54:07.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:54:07.849 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:54:08.490 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:54:10.490 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2629, "optime" : { "t" : 1358380450000, "i" : 7078 }, "optimeDate" : ISODate("2013-01-16T23:54:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2618, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2618, "optime" : { "t" : 1358380446000, "i" : 6078 }, "optimeDate" : ISODate("2013-01-16T23:54:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:09Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2628, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2615, "optime" : { "t" : 1358380446000, "i" : 6078 }, "optimeDate" : ISODate("2013-01-16T23:54:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:10Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2615, "optime" : { "t" : 1358380449000, "i" : 1897 }, "optimeDate" : ISODate("2013-01-16T23:54:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2615, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2627, "optime" : { "t" : 1358380450000, "i" : 7078 }, "optimeDate" : ISODate("2013-01-16T23:54:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:54:12.491 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:54:12.849 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:54:14.491 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2633, "optime" : { "t" : 1358380454000, "i" : 7772 }, "optimeDate" : ISODate("2013-01-16T23:54:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2622, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2622, "optime" : { "t" : 1358380450000, "i" : 7078 }, "optimeDate" : ISODate("2013-01-16T23:54:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:13Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:53:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2632, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2619, "optime" : { "t" : 1358380450000, "i" : 7078 }, "optimeDate" : ISODate("2013-01-16T23:54:10Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2619, "optime" : { "t" : 1358380453000, "i" : 1417 }, "optimeDate" : ISODate("2013-01-16T23:54:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2619, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:13Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2631, "optime" : { "t" : 1358380454000, "i" : 6940 }, "optimeDate" : ISODate("2013-01-16T23:54:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:54:16.491 [conn156] end connection 127.0.0.2:55618 (2 connections now open) m31001| Wed Jan 16 18:54:16.491 [initandlisten] connection accepted from 127.0.0.2:39328 #158 (3 connections now open) m31000| Wed Jan 16 18:54:16.492 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:54:17.850 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:54:18.492 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2638, "optime" : { "t" : 1358380459000, "i" : 368 }, "optimeDate" : ISODate("2013-01-16T23:54:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2627, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2627, "optime" : { "t" : 1358380458000, "i" : 2764 }, "optimeDate" : ISODate("2013-01-16T23:54:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2637, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2624, "optime" : { "t" : 1358380454000, "i" : 7772 }, "optimeDate" : ISODate("2013-01-16T23:54:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:17Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2624, "optime" : { "t" : 1358380457000, "i" : 370 }, "optimeDate" : ISODate("2013-01-16T23:54:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2624, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2636, "optime" : { "t" : 1358380459000, "i" : 368 }, "optimeDate" : ISODate("2013-01-16T23:54:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:54:19.941 [conn97] end connection 127.0.0.4:54350 (4 connections now open) m31000| Wed Jan 16 18:54:19.941 [initandlisten] connection accepted from 127.0.0.4:33647 #98 (5 connections now open) m31000| Wed Jan 16 18:54:20.492 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:54:22.493 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:54:22.850 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2642, "optime" : { "t" : 1358380463000, "i" : 1131 }, "optimeDate" : ISODate("2013-01-16T23:54:23Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2631, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2631, "optime" : { "t" : 1358380459000, "i" : 368 }, "optimeDate" : ISODate("2013-01-16T23:54:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:21Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2641, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2628, "optime" : { "t" : 1358380463000, "i" : 25 }, "optimeDate" : ISODate("2013-01-16T23:54:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2628, "optime" : { "t" : 1358380459000, "i" : 368 }, "optimeDate" : ISODate("2013-01-16T23:54:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2628, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2640, "optime" : { "t" : 1358380463000, "i" : 1131 }, "optimeDate" : ISODate("2013-01-16T23:54:23Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:54:24.493 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:54:26.493 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2646, "optime" : { "t" : 1358380467000, "i" : 2711 }, "optimeDate" : ISODate("2013-01-16T23:54:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2635, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2635, "optime" : { "t" : 1358380463000, "i" : 1131 }, "optimeDate" : ISODate("2013-01-16T23:54:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:25Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2645, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2632, "optime" : { "t" : 1358380467000, "i" : 945 }, "optimeDate" : ISODate("2013-01-16T23:54:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2632, "optime" : { "t" : 1358380463000, "i" : 1131 }, "optimeDate" : ISODate("2013-01-16T23:54:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2632, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:27Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2644, "optime" : { "t" : 1358380467000, "i" : 2711 }, "optimeDate" : ISODate("2013-01-16T23:54:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:54:27.851 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:54:28.494 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:54:29.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:54:30.494 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2650, "optime" : { "t" : 1358380471000, "i" : 4065 }, "optimeDate" : ISODate("2013-01-16T23:54:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2639, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2639, "optime" : { "t" : 1358380470000, "i" : 4588 }, "optimeDate" : ISODate("2013-01-16T23:54:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2649, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2636, "optime" : { "t" : 1358380471000, "i" : 1169 }, "optimeDate" : ISODate("2013-01-16T23:54:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2636, "optime" : { "t" : 1358380467000, "i" : 2711 }, "optimeDate" : ISODate("2013-01-16T23:54:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2636, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:31Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2648, "optime" : { "t" : 1358380471000, "i" : 4065 }, "optimeDate" : ISODate("2013-01-16T23:54:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:54:32.494 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:54:32.852 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:54:33.097 [conn159] end connection 127.0.0.3:42174 (3 connections now open) m31002| Wed Jan 16 18:54:33.097 [initandlisten] connection accepted from 127.0.0.3:36620 #160 (4 connections now open) m31000| Wed Jan 16 18:54:34.495 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2654, "optime" : { "t" : 1358380475000, "i" : 4954 }, "optimeDate" : ISODate("2013-01-16T23:54:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2643, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2643, "optime" : { "t" : 1358380471000, "i" : 4065 }, "optimeDate" : ISODate("2013-01-16T23:54:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:33Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2653, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2640, "optime" : { "t" : 1358380475000, "i" : 638 }, "optimeDate" : ISODate("2013-01-16T23:54:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:35Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2640, "optime" : { "t" : 1358380471000, "i" : 4065 }, "optimeDate" : ISODate("2013-01-16T23:54:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2640, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2652, "optime" : { "t" : 1358380475000, "i" : 4954 }, "optimeDate" : ISODate("2013-01-16T23:54:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:54:35.859 [conn158] end connection 127.0.0.2:60775 (3 connections now open) m31002| Wed Jan 16 18:54:36.359 [initandlisten] connection accepted from 127.0.0.2:37642 #161 (4 connections now open) m31000| Wed Jan 16 18:54:36.495 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:54:37.852 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:54:38.496 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:54:39.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2658, "optime" : { "t" : 1358380479000, "i" : 6362 }, "optimeDate" : ISODate("2013-01-16T23:54:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2647, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2647, "optime" : { "t" : 1358380478000, "i" : 1510 }, "optimeDate" : ISODate("2013-01-16T23:54:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2657, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2644, "optime" : { "t" : 1358380479000, "i" : 480 }, "optimeDate" : ISODate("2013-01-16T23:54:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2644, "optime" : { "t" : 1358380475000, "i" : 4954 }, "optimeDate" : ISODate("2013-01-16T23:54:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2644, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:39Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2656, "optime" : { "t" : 1358380479000, "i" : 5877 }, "optimeDate" : ISODate("2013-01-16T23:54:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:54:40.353 [conn157] end connection 127.0.0.4:53013 (2 connections now open) m31000| Wed Jan 16 18:54:40.496 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:54:40.853 [initandlisten] connection accepted from 127.0.0.4:53276 #159 (3 connections now open) m31000| Wed Jan 16 18:54:42.496 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:54:43.353 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:54:43.723 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:310759 155ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2662, "optime" : { "t" : 1358380483000, "i" : 8030 }, "optimeDate" : ISODate("2013-01-16T23:54:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2651, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2651, "optime" : { "t" : 1358380483000, "i" : 7459 }, "optimeDate" : ISODate("2013-01-16T23:54:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2661, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2648, "optime" : { "t" : 1358380483000, "i" : 994 }, "optimeDate" : ISODate("2013-01-16T23:54:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:43Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2648, "optime" : { "t" : 1358380479000, "i" : 6362 }, "optimeDate" : ISODate("2013-01-16T23:54:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:43Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2648, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2660, "optime" : { "t" : 1358380483000, "i" : 8030 }, "optimeDate" : ISODate("2013-01-16T23:54:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:54:44.497 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:54:46.497 [conn158] end connection 127.0.0.2:39328 (2 connections now open) m31001| Wed Jan 16 18:54:46.497 [initandlisten] connection accepted from 127.0.0.2:53398 #160 (3 connections now open) m31000| Wed Jan 16 18:54:46.498 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2667, "optime" : { "t" : 1358380488000, "i" : 769 }, "optimeDate" : ISODate("2013-01-16T23:54:48Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2656, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2656, "optime" : { "t" : 1358380483000, "i" : 8030 }, "optimeDate" : ISODate("2013-01-16T23:54:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:47Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2666, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2653, "optime" : { "t" : 1358380487000, "i" : 1146 }, "optimeDate" : ISODate("2013-01-16T23:54:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2653, "optime" : { "t" : 1358380487000, "i" : 8682 }, "optimeDate" : ISODate("2013-01-16T23:54:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2653, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2665, "optime" : { "t" : 1358380488000, "i" : 10 }, "optimeDate" : ISODate("2013-01-16T23:54:48Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:54:48.354 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:54:48.498 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:54:49.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:54:49.947 [conn98] end connection 127.0.0.4:33647 (4 connections now open) m31000| Wed Jan 16 18:54:49.947 [initandlisten] connection accepted from 127.0.0.4:53377 #99 (5 connections now open) m31000| Wed Jan 16 18:54:50.498 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2671, "optime" : { "t" : 1358380492000, "i" : 1411 }, "optimeDate" : ISODate("2013-01-16T23:54:52Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2660, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2660, "optime" : { "t" : 1358380491000, "i" : 2363 }, "optimeDate" : ISODate("2013-01-16T23:54:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:51Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2670, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2657, "optime" : { "t" : 1358380491000, "i" : 20 }, "optimeDate" : ISODate("2013-01-16T23:54:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2657, "optime" : { "t" : 1358380491000, "i" : 8145 }, "optimeDate" : ISODate("2013-01-16T23:54:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2657, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2669, "optime" : { "t" : 1358380492000, "i" : 1411 }, "optimeDate" : ISODate("2013-01-16T23:54:52Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:54:52.499 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:54:53.355 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:54:54.499 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2675, "optime" : { "t" : 1358380496000, "i" : 3208 }, "optimeDate" : ISODate("2013-01-16T23:54:56Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2664, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2664, "optime" : { "t" : 1358380496000, "i" : 2630 }, "optimeDate" : ISODate("2013-01-16T23:54:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2674, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2661, "optime" : { "t" : 1358380492000, "i" : 1411 }, "optimeDate" : ISODate("2013-01-16T23:54:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:55Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:54:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2661, "optime" : { "t" : 1358380495000, "i" : 6310 }, "optimeDate" : ISODate("2013-01-16T23:54:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2661, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2673, "optime" : { "t" : 1358380496000, "i" : 2630 }, "optimeDate" : ISODate("2013-01-16T23:54:56Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:54:56.500 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:54:58.355 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:54:58.500 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2679, "optime" : { "t" : 1358380500000, "i" : 4313 }, "optimeDate" : ISODate("2013-01-16T23:55:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2668, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2668, "optime" : { "t" : 1358380496000, "i" : 3208 }, "optimeDate" : ISODate("2013-01-16T23:54:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:59Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2678, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2665, "optime" : { "t" : 1358380496000, "i" : 3208 }, "optimeDate" : ISODate("2013-01-16T23:54:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2665, "optime" : { "t" : 1358380499000, "i" : 4643 }, "optimeDate" : ISODate("2013-01-16T23:54:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2665, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:54:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2677, "optime" : { "t" : 1358380500000, "i" : 3967 }, "optimeDate" : ISODate("2013-01-16T23:55:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:55:00.500 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:55:02.501 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:55:03.102 [conn160] end connection 127.0.0.3:36620 (3 connections now open) m31002| Wed Jan 16 18:55:03.102 [initandlisten] connection accepted from 127.0.0.3:36431 #162 (4 connections now open) m31002| Wed Jan 16 18:55:03.356 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:55:04.501 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2683, "optime" : { "t" : 1358380504000, "i" : 5540 }, "optimeDate" : ISODate("2013-01-16T23:55:04Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2672, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2672, "optime" : { "t" : 1358380503000, "i" : 2975 }, "optimeDate" : ISODate("2013-01-16T23:55:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:03Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2682, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2669, "optime" : { "t" : 1358380500000, "i" : 4313 }, "optimeDate" : ISODate("2013-01-16T23:55:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:03Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2669, "optime" : { "t" : 1358380503000, "i" : 4045 }, "optimeDate" : ISODate("2013-01-16T23:55:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2669, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2681, "optime" : { "t" : 1358380504000, "i" : 5431 }, "optimeDate" : ISODate("2013-01-16T23:55:04Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:55:06.501 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:55:08.357 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:55:08.502 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2687, "optime" : { "t" : 1358380508000, "i" : 7187 }, "optimeDate" : ISODate("2013-01-16T23:55:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2676, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2676, "optime" : { "t" : 1358380504000, "i" : 5540 }, "optimeDate" : ISODate("2013-01-16T23:55:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:07Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2686, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2673, "optime" : { "t" : 1358380504000, "i" : 5540 }, "optimeDate" : ISODate("2013-01-16T23:55:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:08Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2673, "optime" : { "t" : 1358380507000, "i" : 2448 }, "optimeDate" : ISODate("2013-01-16T23:55:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2673, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2685, "optime" : { "t" : 1358380508000, "i" : 7039 }, "optimeDate" : ISODate("2013-01-16T23:55:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:55:10.502 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:55:11.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:55:12.502 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2691, "optime" : { "t" : 1358380512000, "i" : 8441 }, "optimeDate" : ISODate("2013-01-16T23:55:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2680, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2680, "optime" : { "t" : 1358380508000, "i" : 7187 }, "optimeDate" : ISODate("2013-01-16T23:55:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:11Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2690, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2677, "optime" : { "t" : 1358380508000, "i" : 7187 }, "optimeDate" : ISODate("2013-01-16T23:55:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2677, "optime" : { "t" : 1358380511000, "i" : 454 }, "optimeDate" : ISODate("2013-01-16T23:55:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2677, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2689, "optime" : { "t" : 1358380512000, "i" : 8441 }, "optimeDate" : ISODate("2013-01-16T23:55:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:55:13.357 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:55:13.864 [conn161] end connection 127.0.0.2:37642 (3 connections now open) m31002| Wed Jan 16 18:55:14.364 [initandlisten] connection accepted from 127.0.0.2:41784 #163 (4 connections now open) m31000| Wed Jan 16 18:55:14.503 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:55:16.503 [conn160] end connection 127.0.0.2:53398 (2 connections now open) m31001| Wed Jan 16 18:55:16.503 [initandlisten] connection accepted from 127.0.0.2:45873 #161 (3 connections now open) m31000| Wed Jan 16 18:55:16.503 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2696, "optime" : { "t" : 1358380517000, "i" : 1636 }, "optimeDate" : ISODate("2013-01-16T23:55:17Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2685, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2685, "optime" : { "t" : 1358380516000, "i" : 6595 }, "optimeDate" : ISODate("2013-01-16T23:55:16Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2695, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2682, "optime" : { "t" : 1358380517000, "i" : 1021 }, "optimeDate" : ISODate("2013-01-16T23:55:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2682, "optime" : { "t" : 1358380512000, "i" : 8441 }, "optimeDate" : ISODate("2013-01-16T23:55:12Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2682, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2694, "optime" : { "t" : 1358380517000, "i" : 1636 }, "optimeDate" : ISODate("2013-01-16T23:55:17Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:55:18.358 [conn159] end connection 127.0.0.4:53276 (2 connections now open) m31000| Wed Jan 16 18:55:18.503 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:55:18.858 [initandlisten] connection accepted from 127.0.0.4:43836 #162 (3 connections now open) m31002| Wed Jan 16 18:55:18.858 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:55:19.952 [conn99] end connection 127.0.0.4:53377 (4 connections now open) m31000| Wed Jan 16 18:55:19.952 [initandlisten] connection accepted from 127.0.0.4:49810 #100 (5 connections now open) m31000| Wed Jan 16 18:55:20.504 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2700, "optime" : { "t" : 1358380521000, "i" : 3204 }, "optimeDate" : ISODate("2013-01-16T23:55:21Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2689, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2689, "optime" : { "t" : 1358380517000, "i" : 1636 }, "optimeDate" : ISODate("2013-01-16T23:55:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:19Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2699, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2686, "optime" : { "t" : 1358380521000, "i" : 983 }, "optimeDate" : ISODate("2013-01-16T23:55:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2686, "optime" : { "t" : 1358380517000, "i" : 1636 }, "optimeDate" : ISODate("2013-01-16T23:55:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2686, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:21Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2698, "optime" : { "t" : 1358380521000, "i" : 2466 }, "optimeDate" : ISODate("2013-01-16T23:55:21Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:55:21.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:55:22.504 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:55:23.859 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:55:24.504 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2704, "optime" : { "t" : 1358380525000, "i" : 4077 }, "optimeDate" : ISODate("2013-01-16T23:55:25Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2693, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2693, "optime" : { "t" : 1358380524000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:55:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2703, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2690, "optime" : { "t" : 1358380525000, "i" : 727 }, "optimeDate" : ISODate("2013-01-16T23:55:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:25Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2690, "optime" : { "t" : 1358380521000, "i" : 3204 }, "optimeDate" : ISODate("2013-01-16T23:55:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2690, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2702, "optime" : { "t" : 1358380525000, "i" : 4077 }, "optimeDate" : ISODate("2013-01-16T23:55:25Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:55:26.505 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:55:28.505 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:55:28.860 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2708, "optime" : { "t" : 1358380529000, "i" : 5241 }, "optimeDate" : ISODate("2013-01-16T23:55:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2697, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2697, "optime" : { "t" : 1358380529000, "i" : 2937 }, "optimeDate" : ISODate("2013-01-16T23:55:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:54:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2707, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2694, "optime" : { "t" : 1358380529000, "i" : 197 }, "optimeDate" : ISODate("2013-01-16T23:55:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2694, "optime" : { "t" : 1358380525000, "i" : 4077 }, "optimeDate" : ISODate("2013-01-16T23:55:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:29Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2694, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:29Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2706, "optime" : { "t" : 1358380529000, "i" : 5241 }, "optimeDate" : ISODate("2013-01-16T23:55:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:55:30.505 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:55:32.506 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:55:33.107 [conn162] end connection 127.0.0.3:36431 (3 connections now open) m31002| Wed Jan 16 18:55:33.108 [initandlisten] connection accepted from 127.0.0.3:59792 #164 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2712, "optime" : { "t" : 1358380533000, "i" : 6972 }, "optimeDate" : ISODate("2013-01-16T23:55:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2701, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2701, "optime" : { "t" : 1358380529000, "i" : 5241 }, "optimeDate" : ISODate("2013-01-16T23:55:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:31Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2711, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2698, "optime" : { "t" : 1358380533000, "i" : 997 }, "optimeDate" : ISODate("2013-01-16T23:55:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2698, "optime" : { "t" : 1358380529000, "i" : 5241 }, "optimeDate" : ISODate("2013-01-16T23:55:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2698, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:33Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2710, "optime" : { "t" : 1358380533000, "i" : 6356 }, "optimeDate" : ISODate("2013-01-16T23:55:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:55:33.860 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:55:34.506 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:55:36.507 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2716, "optime" : { "t" : 1358380537000, "i" : 7877 }, "optimeDate" : ISODate("2013-01-16T23:55:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2705, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2705, "optime" : { "t" : 1358380536000, "i" : 931 }, "optimeDate" : ISODate("2013-01-16T23:55:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2715, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2702, "optime" : { "t" : 1358380537000, "i" : 527 }, "optimeDate" : ISODate("2013-01-16T23:55:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2702, "optime" : { "t" : 1358380533000, "i" : 6972 }, "optimeDate" : ISODate("2013-01-16T23:55:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2702, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2714, "optime" : { "t" : 1358380537000, "i" : 7741 }, "optimeDate" : ISODate("2013-01-16T23:55:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:55:38.507 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:55:38.861 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:55:40.507 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2721, "optime" : { "t" : 1358380542000, "i" : 379 }, "optimeDate" : ISODate("2013-01-16T23:55:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2710, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2710, "optime" : { "t" : 1358380541000, "i" : 7555 }, "optimeDate" : ISODate("2013-01-16T23:55:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:41Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2720, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2707, "optime" : { "t" : 1358380541000, "i" : 918 }, "optimeDate" : ISODate("2013-01-16T23:55:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:41Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2707, "optime" : { "t" : 1358380541000, "i" : 8841 }, "optimeDate" : ISODate("2013-01-16T23:55:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2707, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2719, "optime" : { "t" : 1358380542000, "i" : 379 }, "optimeDate" : ISODate("2013-01-16T23:55:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:55:42.508 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:55:43.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:55:43.862 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:55:44.508 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2725, "optime" : { "t" : 1358380546000, "i" : 2552 }, "optimeDate" : ISODate("2013-01-16T23:55:46Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2714, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2714, "optime" : { "t" : 1358380542000, "i" : 379 }, "optimeDate" : ISODate("2013-01-16T23:55:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:45Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2724, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2711, "optime" : { "t" : 1358380545000, "i" : 160 }, "optimeDate" : ISODate("2013-01-16T23:55:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2711, "optime" : { "t" : 1358380545000, "i" : 7264 }, "optimeDate" : ISODate("2013-01-16T23:55:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2711, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2723, "optime" : { "t" : 1358380546000, "i" : 2552 }, "optimeDate" : ISODate("2013-01-16T23:55:46Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:55:46.508 [conn161] end connection 127.0.0.2:45873 (2 connections now open) m31001| Wed Jan 16 18:55:46.509 [initandlisten] connection accepted from 127.0.0.2:58106 #163 (3 connections now open) m31000| Wed Jan 16 18:55:46.509 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:55:48.509 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:55:48.862 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:55:49.957 [conn100] end connection 127.0.0.4:49810 (4 connections now open) m31000| Wed Jan 16 18:55:49.958 [initandlisten] connection accepted from 127.0.0.4:49343 #101 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2729, "optime" : { "t" : 1358380550000, "i" : 4209 }, "optimeDate" : ISODate("2013-01-16T23:55:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2718, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2718, "optime" : { "t" : 1358380549000, "i" : 162 }, "optimeDate" : ISODate("2013-01-16T23:55:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:49Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2728, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2715, "optime" : { "t" : 1358380546000, "i" : 2552 }, "optimeDate" : ISODate("2013-01-16T23:55:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2715, "optime" : { "t" : 1358380549000, "i" : 5289 }, "optimeDate" : ISODate("2013-01-16T23:55:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2715, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2727, "optime" : { "t" : 1358380550000, "i" : 3889 }, "optimeDate" : ISODate("2013-01-16T23:55:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:55:50.510 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:55:51.869 [conn163] end connection 127.0.0.2:41784 (3 connections now open) m31002| Wed Jan 16 18:55:52.370 [initandlisten] connection accepted from 127.0.0.2:37140 #165 (4 connections now open) m31000| Wed Jan 16 18:55:52.510 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:55:53.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:55:53.638 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:234128 117ms m31002| Wed Jan 16 18:55:53.863 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:55:54.510 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2733, "optime" : { "t" : 1358380554000, "i" : 5398 }, "optimeDate" : ISODate("2013-01-16T23:55:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2722, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2722, "optime" : { "t" : 1358380550000, "i" : 4209 }, "optimeDate" : ISODate("2013-01-16T23:55:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:53Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2732, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2719, "optime" : { "t" : 1358380550000, "i" : 4209 }, "optimeDate" : ISODate("2013-01-16T23:55:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:53Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2719, "optime" : { "t" : 1358380553000, "i" : 3764 }, "optimeDate" : ISODate("2013-01-16T23:55:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2719, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2731, "optime" : { "t" : 1358380554000, "i" : 5398 }, "optimeDate" : ISODate("2013-01-16T23:55:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:55:56.363 [conn162] end connection 127.0.0.4:43836 (2 connections now open) m31000| Wed Jan 16 18:55:56.511 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:55:56.863 [initandlisten] connection accepted from 127.0.0.4:33071 #164 (3 connections now open) m31000| Wed Jan 16 18:55:58.511 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:55:58.656 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:310655 155ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2737, "optime" : { "t" : 1358380558000, "i" : 6391 }, "optimeDate" : ISODate("2013-01-16T23:55:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2726, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2726, "optime" : { "t" : 1358380554000, "i" : 5398 }, "optimeDate" : ISODate("2013-01-16T23:55:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:57Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2736, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2723, "optime" : { "t" : 1358380554000, "i" : 5398 }, "optimeDate" : ISODate("2013-01-16T23:55:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:55:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2723, "optime" : { "t" : 1358380557000, "i" : 2926 }, "optimeDate" : ISODate("2013-01-16T23:55:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2723, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:55:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2735, "optime" : { "t" : 1358380558000, "i" : 6391 }, "optimeDate" : ISODate("2013-01-16T23:55:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:55:59.364 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:56:00.512 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:56:02.512 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2741, "optime" : { "t" : 1358380562000, "i" : 7679 }, "optimeDate" : ISODate("2013-01-16T23:56:02Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2730, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2730, "optime" : { "t" : 1358380562000, "i" : 3502 }, "optimeDate" : ISODate("2013-01-16T23:56:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:55:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2740, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2727, "optime" : { "t" : 1358380558000, "i" : 6391 }, "optimeDate" : ISODate("2013-01-16T23:55:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:01Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2727, "optime" : { "t" : 1358380561000, "i" : 2004 }, "optimeDate" : ISODate("2013-01-16T23:56:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2727, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2739, "optime" : { "t" : 1358380562000, "i" : 7679 }, "optimeDate" : ISODate("2013-01-16T23:56:02Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:56:03.113 [conn164] end connection 127.0.0.3:59792 (3 connections now open) m31002| Wed Jan 16 18:56:03.113 [initandlisten] connection accepted from 127.0.0.3:49833 #166 (4 connections now open) m31002| Wed Jan 16 18:56:04.365 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:56:04.512 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:56:06.512 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2746, "optime" : { "t" : 1358380567000, "i" : 415 }, "optimeDate" : ISODate("2013-01-16T23:56:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2735, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2735, "optime" : { "t" : 1358380562000, "i" : 7679 }, "optimeDate" : ISODate("2013-01-16T23:56:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:05Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2745, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2732, "optime" : { "t" : 1358380562000, "i" : 7679 }, "optimeDate" : ISODate("2013-01-16T23:56:02Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:06Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2732, "optime" : { "t" : 1358380565000, "i" : 208 }, "optimeDate" : ISODate("2013-01-16T23:56:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2732, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2744, "optime" : { "t" : 1358380567000, "i" : 365 }, "optimeDate" : ISODate("2013-01-16T23:56:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:56:08.513 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:56:09.365 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:56:10.513 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2750, "optime" : { "t" : 1358380571000, "i" : 2392 }, "optimeDate" : ISODate("2013-01-16T23:56:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2739, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2739, "optime" : { "t" : 1358380567000, "i" : 415 }, "optimeDate" : ISODate("2013-01-16T23:56:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:09Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2749, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2736, "optime" : { "t" : 1358380571000, "i" : 1166 }, "optimeDate" : ISODate("2013-01-16T23:56:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2736, "optime" : { "t" : 1358380567000, "i" : 415 }, "optimeDate" : ISODate("2013-01-16T23:56:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2736, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2748, "optime" : { "t" : 1358380571000, "i" : 2392 }, "optimeDate" : ISODate("2013-01-16T23:56:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:56:12.514 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:56:14.366 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:56:14.514 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2754, "optime" : { "t" : 1358380575000, "i" : 3542 }, "optimeDate" : ISODate("2013-01-16T23:56:15Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2743, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2743, "optime" : { "t" : 1358380574000, "i" : 5183 }, "optimeDate" : ISODate("2013-01-16T23:56:14Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2753, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2740, "optime" : { "t" : 1358380575000, "i" : 1037 }, "optimeDate" : ISODate("2013-01-16T23:56:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2740, "optime" : { "t" : 1358380571000, "i" : 2392 }, "optimeDate" : ISODate("2013-01-16T23:56:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2740, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:15Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2752, "optime" : { "t" : 1358380575000, "i" : 3542 }, "optimeDate" : ISODate("2013-01-16T23:56:15Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:56:15.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 18:56:16.514 [conn163] end connection 127.0.0.2:58106 (2 connections now open) m31001| Wed Jan 16 18:56:16.514 [initandlisten] connection accepted from 127.0.0.2:50137 #165 (3 connections now open) m31000| Wed Jan 16 18:56:16.515 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:56:18.515 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:56:19.366 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:56:19.458 [journal] old journal file will be removed: /data/db/testReplSet-0/journal/j._3 STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:56:19.535 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:331765 165ms { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2758, "optime" : { "t" : 1358380579000, "i" : 4916 }, "optimeDate" : ISODate("2013-01-16T23:56:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2747, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2747, "optime" : { "t" : 1358380575000, "i" : 3542 }, "optimeDate" : ISODate("2013-01-16T23:56:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:17Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2757, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2744, "optime" : { "t" : 1358380579000, "i" : 379 }, "optimeDate" : ISODate("2013-01-16T23:56:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:19Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2744, "optime" : { "t" : 1358380575000, "i" : 3542 }, "optimeDate" : ISODate("2013-01-16T23:56:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2744, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2756, "optime" : { "t" : 1358380579000, "i" : 3410 }, "optimeDate" : ISODate("2013-01-16T23:56:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:56:19.963 [conn101] end connection 127.0.0.4:49343 (4 connections now open) m31000| Wed Jan 16 18:56:19.963 [initandlisten] connection accepted from 127.0.0.4:37865 #102 (5 connections now open) m31000| Wed Jan 16 18:56:20.515 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:56:22.516 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2762, "optime" : { "t" : 1358380583000, "i" : 6828 }, "optimeDate" : ISODate("2013-01-16T23:56:23Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2751, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2751, "optime" : { "t" : 1358380579000, "i" : 4916 }, "optimeDate" : ISODate("2013-01-16T23:56:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2761, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2748, "optime" : { "t" : 1358380583000, "i" : 1171 }, "optimeDate" : ISODate("2013-01-16T23:56:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2748, "optime" : { "t" : 1358380579000, "i" : 4916 }, "optimeDate" : ISODate("2013-01-16T23:56:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2748, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2760, "optime" : { "t" : 1358380583000, "i" : 6540 }, "optimeDate" : ISODate("2013-01-16T23:56:23Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:56:24.367 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:56:24.516 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:56:25.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:56:26.516 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2766, "optime" : { "t" : 1358380587000, "i" : 7735 }, "optimeDate" : ISODate("2013-01-16T23:56:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2755, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2755, "optime" : { "t" : 1358380587000, "i" : 3004 }, "optimeDate" : ISODate("2013-01-16T23:56:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2765, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2752, "optime" : { "t" : 1358380587000, "i" : 571 }, "optimeDate" : ISODate("2013-01-16T23:56:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2752, "optime" : { "t" : 1358380583000, "i" : 6828 }, "optimeDate" : ISODate("2013-01-16T23:56:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:27Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2752, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:27Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2764, "optime" : { "t" : 1358380587000, "i" : 7115 }, "optimeDate" : ISODate("2013-01-16T23:56:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:56:28.517 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:56:29.368 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:56:29.875 [conn165] end connection 127.0.0.2:37140 (3 connections now open) m31002| Wed Jan 16 18:56:30.375 [initandlisten] connection accepted from 127.0.0.2:40036 #167 (4 connections now open) m31000| Wed Jan 16 18:56:30.517 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2771, "optime" : { "t" : 1358380592000, "i" : 578 }, "optimeDate" : ISODate("2013-01-16T23:56:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2760, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2760, "optime" : { "t" : 1358380587000, "i" : 7735 }, "optimeDate" : ISODate("2013-01-16T23:56:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:31Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2770, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2757, "optime" : { "t" : 1358380591000, "i" : 562 }, "optimeDate" : ISODate("2013-01-16T23:56:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:31Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2757, "optime" : { "t" : 1358380591000, "i" : 8600 }, "optimeDate" : ISODate("2013-01-16T23:56:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2757, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2769, "optime" : { "t" : 1358380592000, "i" : 578 }, "optimeDate" : ISODate("2013-01-16T23:56:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:56:32.518 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:56:33.118 [conn166] end connection 127.0.0.3:49833 (3 connections now open) m31002| Wed Jan 16 18:56:33.118 [initandlisten] connection accepted from 127.0.0.3:46327 #168 (4 connections now open) m31001| Wed Jan 16 18:56:34.368 [conn164] end connection 127.0.0.4:33071 (2 connections now open) m31000| Wed Jan 16 18:56:34.518 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:56:34.868 [initandlisten] connection accepted from 127.0.0.4:40185 #166 (3 connections now open) m31002| Wed Jan 16 18:56:34.869 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2775, "optime" : { "t" : 1358380596000, "i" : 1996 }, "optimeDate" : ISODate("2013-01-16T23:56:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2764, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2764, "optime" : { "t" : 1358380595000, "i" : 2455 }, "optimeDate" : ISODate("2013-01-16T23:56:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:35Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2774, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2761, "optime" : { "t" : 1358380595000, "i" : 449 }, "optimeDate" : ISODate("2013-01-16T23:56:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2761, "optime" : { "t" : 1358380595000, "i" : 7412 }, "optimeDate" : ISODate("2013-01-16T23:56:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2761, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:35Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2773, "optime" : { "t" : 1358380596000, "i" : 1996 }, "optimeDate" : ISODate("2013-01-16T23:56:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:56:36.518 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:56:38.519 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:56:39.869 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2779, "optime" : { "t" : 1358380600000, "i" : 3350 }, "optimeDate" : ISODate("2013-01-16T23:56:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2768, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2768, "optime" : { "t" : 1358380600000, "i" : 2722 }, "optimeDate" : ISODate("2013-01-16T23:56:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2778, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2765, "optime" : { "t" : 1358380596000, "i" : 1996 }, "optimeDate" : ISODate("2013-01-16T23:56:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:39Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2765, "optime" : { "t" : 1358380599000, "i" : 5798 }, "optimeDate" : ISODate("2013-01-16T23:56:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2765, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2777, "optime" : { "t" : 1358380600000, "i" : 3350 }, "optimeDate" : ISODate("2013-01-16T23:56:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:56:40.519 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:56:42.520 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:56:44.520 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2783, "optime" : { "t" : 1358380604000, "i" : 4998 }, "optimeDate" : ISODate("2013-01-16T23:56:44Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2772, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2772, "optime" : { "t" : 1358380600000, "i" : 3350 }, "optimeDate" : ISODate("2013-01-16T23:56:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:43Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2782, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2769, "optime" : { "t" : 1358380600000, "i" : 3350 }, "optimeDate" : ISODate("2013-01-16T23:56:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2769, "optime" : { "t" : 1358380603000, "i" : 4706 }, "optimeDate" : ISODate("2013-01-16T23:56:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2769, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:43Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2781, "optime" : { "t" : 1358380604000, "i" : 4998 }, "optimeDate" : ISODate("2013-01-16T23:56:44Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:56:44.870 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:56:46.520 [conn165] end connection 127.0.0.2:50137 (2 connections now open) m31001| Wed Jan 16 18:56:46.520 [initandlisten] connection accepted from 127.0.0.2:40867 #167 (3 connections now open) m31000| Wed Jan 16 18:56:46.521 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:56:47.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:56:48.521 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2787, "optime" : { "t" : 1358380608000, "i" : 6724 }, "optimeDate" : ISODate("2013-01-16T23:56:48Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2776, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2776, "optime" : { "t" : 1358380607000, "i" : 2226 }, "optimeDate" : ISODate("2013-01-16T23:56:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:47Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2786, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2773, "optime" : { "t" : 1358380604000, "i" : 4998 }, "optimeDate" : ISODate("2013-01-16T23:56:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:47Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2773, "optime" : { "t" : 1358380607000, "i" : 3275 }, "optimeDate" : ISODate("2013-01-16T23:56:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2773, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2785, "optime" : { "t" : 1358380608000, "i" : 6028 }, "optimeDate" : ISODate("2013-01-16T23:56:48Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:56:49.871 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:56:49.968 [conn102] end connection 127.0.0.4:37865 (4 connections now open) m31000| Wed Jan 16 18:56:49.968 [initandlisten] connection accepted from 127.0.0.4:40568 #103 (5 connections now open) m31000| Wed Jan 16 18:56:50.521 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:56:51.800 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:287920 143ms m31000| Wed Jan 16 18:56:52.521 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2791, "optime" : { "t" : 1358380612000, "i" : 6619 }, "optimeDate" : ISODate("2013-01-16T23:56:52Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2780, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2780, "optime" : { "t" : 1358380608000, "i" : 6724 }, "optimeDate" : ISODate("2013-01-16T23:56:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:51Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2790, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2777, "optime" : { "t" : 1358380608000, "i" : 6724 }, "optimeDate" : ISODate("2013-01-16T23:56:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:52Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2777, "optime" : { "t" : 1358380611000, "i" : 2668 }, "optimeDate" : ISODate("2013-01-16T23:56:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2777, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2789, "optime" : { "t" : 1358380612000, "i" : 6619 }, "optimeDate" : ISODate("2013-01-16T23:56:52Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:56:54.522 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:56:54.871 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:56:56.522 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2795, "optime" : { "t" : 1358380616000, "i" : 8262 }, "optimeDate" : ISODate("2013-01-16T23:56:56Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2784, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2784, "optime" : { "t" : 1358380612000, "i" : 6619 }, "optimeDate" : ISODate("2013-01-16T23:56:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:55Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2794, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2781, "optime" : { "t" : 1358380612000, "i" : 6619 }, "optimeDate" : ISODate("2013-01-16T23:56:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:56:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2781, "optime" : { "t" : 1358380615000, "i" : 1436 }, "optimeDate" : ISODate("2013-01-16T23:56:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2781, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2793, "optime" : { "t" : 1358380616000, "i" : 8262 }, "optimeDate" : ISODate("2013-01-16T23:56:56Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:56:57.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:56:58.523 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:56:59.872 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:57:00.523 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2799, "optime" : { "t" : 1358380620000, "i" : 9814 }, "optimeDate" : ISODate("2013-01-16T23:57:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2788, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2788, "optime" : { "t" : 1358380620000, "i" : 3807 }, "optimeDate" : ISODate("2013-01-16T23:57:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2799, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2786, "optime" : { "t" : 1358380616000, "i" : 8262 }, "optimeDate" : ISODate("2013-01-16T23:56:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:56:59Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2786, "optime" : { "t" : 1358380619000, "i" : 85 }, "optimeDate" : ISODate("2013-01-16T23:56:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2786, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2798, "optime" : { "t" : 1358380620000, "i" : 9687 }, "optimeDate" : ISODate("2013-01-16T23:57:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:57:02.523 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:57:03.124 [conn168] end connection 127.0.0.3:46327 (3 connections now open) m31002| Wed Jan 16 18:57:03.124 [initandlisten] connection accepted from 127.0.0.3:42908 #169 (4 connections now open) m31000| Wed Jan 16 18:57:04.524 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:57:04.873 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2804, "optime" : { "t" : 1358380625000, "i" : 1002 }, "optimeDate" : ISODate("2013-01-16T23:57:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2793, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2793, "optime" : { "t" : 1358380620000, "i" : 9814 }, "optimeDate" : ISODate("2013-01-16T23:57:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:03Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:56:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2803, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2790, "optime" : { "t" : 1358380620000, "i" : 9814 }, "optimeDate" : ISODate("2013-01-16T23:57:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:04Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2790, "optime" : { "t" : 1358380620000, "i" : 9814 }, "optimeDate" : ISODate("2013-01-16T23:57:00Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2790, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2802, "optime" : { "t" : 1358380625000, "i" : 1002 }, "optimeDate" : ISODate("2013-01-16T23:57:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:57:06.524 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:57:07.881 [conn167] end connection 127.0.0.2:40036 (3 connections now open) m31002| Wed Jan 16 18:57:08.381 [initandlisten] connection accepted from 127.0.0.2:50393 #170 (4 connections now open) m31000| Wed Jan 16 18:57:08.524 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:57:08.763 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:434980 217ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2808, "optime" : { "t" : 1358380629000, "i" : 1188 }, "optimeDate" : ISODate("2013-01-16T23:57:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2797, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2797, "optime" : { "t" : 1358380628000, "i" : 2393 }, "optimeDate" : ISODate("2013-01-16T23:57:08Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2807, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2794, "optime" : { "t" : 1358380629000, "i" : 239 }, "optimeDate" : ISODate("2013-01-16T23:57:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2794, "optime" : { "t" : 1358380625000, "i" : 1002 }, "optimeDate" : ISODate("2013-01-16T23:57:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2794, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:09Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2806, "optime" : { "t" : 1358380629000, "i" : 1116 }, "optimeDate" : ISODate("2013-01-16T23:57:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:57:09.873 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:57:10.525 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:57:12.374 [conn166] end connection 127.0.0.4:40185 (2 connections now open) m31000| Wed Jan 16 18:57:12.525 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:57:12.874 [initandlisten] connection accepted from 127.0.0.4:55349 #168 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2812, "optime" : { "t" : 1358380633000, "i" : 2752 }, "optimeDate" : ISODate("2013-01-16T23:57:13Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2801, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2801, "optime" : { "t" : 1358380629000, "i" : 1188 }, "optimeDate" : ISODate("2013-01-16T23:57:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:11Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2811, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2798, "optime" : { "t" : 1358380633000, "i" : 1254 }, "optimeDate" : ISODate("2013-01-16T23:57:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2798, "optime" : { "t" : 1358380629000, "i" : 1188 }, "optimeDate" : ISODate("2013-01-16T23:57:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2798, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:13Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2810, "optime" : { "t" : 1358380633000, "i" : 2752 }, "optimeDate" : ISODate("2013-01-16T23:57:13Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:57:14.525 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:57:15.374 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:57:16.526 [conn167] end connection 127.0.0.2:40867 (2 connections now open) m31001| Wed Jan 16 18:57:16.526 [initandlisten] connection accepted from 127.0.0.2:49463 #169 (3 connections now open) m31000| Wed Jan 16 18:57:16.526 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:57:16.543 [journal] old journal file will be removed: /data/db/testReplSet-2/journal/j._3 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2816, "optime" : { "t" : 1358380637000, "i" : 3901 }, "optimeDate" : ISODate("2013-01-16T23:57:17Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2805, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2805, "optime" : { "t" : 1358380633000, "i" : 2752 }, "optimeDate" : ISODate("2013-01-16T23:57:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:15Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2815, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2802, "optime" : { "t" : 1358380637000, "i" : 1300 }, "optimeDate" : ISODate("2013-01-16T23:57:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2802, "optime" : { "t" : 1358380633000, "i" : 2752 }, "optimeDate" : ISODate("2013-01-16T23:57:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2802, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2814, "optime" : { "t" : 1358380637000, "i" : 3901 }, "optimeDate" : ISODate("2013-01-16T23:57:17Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:57:18.526 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:57:19.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:57:19.974 [conn103] end connection 127.0.0.4:40568 (4 connections now open) m31000| Wed Jan 16 18:57:19.974 [initandlisten] connection accepted from 127.0.0.4:44952 #104 (5 connections now open) m31002| Wed Jan 16 18:57:20.375 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:57:20.526 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2820, "optime" : { "t" : 1358380641000, "i" : 4906 }, "optimeDate" : ISODate("2013-01-16T23:57:21Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2809, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2809, "optime" : { "t" : 1358380640000, "i" : 4045 }, "optimeDate" : ISODate("2013-01-16T23:57:20Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2819, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2806, "optime" : { "t" : 1358380641000, "i" : 645 }, "optimeDate" : ISODate("2013-01-16T23:57:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2806, "optime" : { "t" : 1358380637000, "i" : 3901 }, "optimeDate" : ISODate("2013-01-16T23:57:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2806, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:21Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2818, "optime" : { "t" : 1358380641000, "i" : 4906 }, "optimeDate" : ISODate("2013-01-16T23:57:21Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:57:22.527 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:57:24.527 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:57:25.376 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2824, "optime" : { "t" : 1358380645000, "i" : 6480 }, "optimeDate" : ISODate("2013-01-16T23:57:25Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2813, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2813, "optime" : { "t" : 1358380641000, "i" : 4906 }, "optimeDate" : ISODate("2013-01-16T23:57:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:23Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2823, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2810, "optime" : { "t" : 1358380645000, "i" : 918 }, "optimeDate" : ISODate("2013-01-16T23:57:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:25Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:25Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2810, "optime" : { "t" : 1358380641000, "i" : 4906 }, "optimeDate" : ISODate("2013-01-16T23:57:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2810, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2822, "optime" : { "t" : 1358380645000, "i" : 5981 }, "optimeDate" : ISODate("2013-01-16T23:57:25Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:57:26.528 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:57:28.528 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:57:29.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2828, "optime" : { "t" : 1358380649000, "i" : 7766 }, "optimeDate" : ISODate("2013-01-16T23:57:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2817, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2817, "optime" : { "t" : 1358380645000, "i" : 6480 }, "optimeDate" : ISODate("2013-01-16T23:57:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2827, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2814, "optime" : { "t" : 1358380649000, "i" : 790 }, "optimeDate" : ISODate("2013-01-16T23:57:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2814, "optime" : { "t" : 1358380645000, "i" : 6480 }, "optimeDate" : ISODate("2013-01-16T23:57:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2814, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:29Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2826, "optime" : { "t" : 1358380649000, "i" : 7319 }, "optimeDate" : ISODate("2013-01-16T23:57:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:57:30.376 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:57:30.529 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:57:32.529 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:57:33.129 [conn169] end connection 127.0.0.3:42908 (3 connections now open) m31002| Wed Jan 16 18:57:33.129 [initandlisten] connection accepted from 127.0.0.3:49866 #171 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2833, "optime" : { "t" : 1358380654000, "i" : 280 }, "optimeDate" : ISODate("2013-01-16T23:57:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2822, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2822, "optime" : { "t" : 1358380653000, "i" : 3573 }, "optimeDate" : ISODate("2013-01-16T23:57:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:33Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2832, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2819, "optime" : { "t" : 1358380653000, "i" : 1296 }, "optimeDate" : ISODate("2013-01-16T23:57:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2819, "optime" : { "t" : 1358380653000, "i" : 8821 }, "optimeDate" : ISODate("2013-01-16T23:57:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2819, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:33Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2831, "optime" : { "t" : 1358380654000, "i" : 280 }, "optimeDate" : ISODate("2013-01-16T23:57:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:57:34.529 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:57:35.377 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:57:36.530 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2837, "optime" : { "t" : 1358380658000, "i" : 1813 }, "optimeDate" : ISODate("2013-01-16T23:57:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2826, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2826, "optime" : { "t" : 1358380654000, "i" : 280 }, "optimeDate" : ISODate("2013-01-16T23:57:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:37Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2836, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2823, "optime" : { "t" : 1358380657000, "i" : 179 }, "optimeDate" : ISODate("2013-01-16T23:57:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:37Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2823, "optime" : { "t" : 1358380657000, "i" : 7536 }, "optimeDate" : ISODate("2013-01-16T23:57:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2823, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2835, "optime" : { "t" : 1358380658000, "i" : 1813 }, "optimeDate" : ISODate("2013-01-16T23:57:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:57:38.530 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:57:40.378 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:57:40.531 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2841, "optime" : { "t" : 1358380662000, "i" : 3655 }, "optimeDate" : ISODate("2013-01-16T23:57:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2830, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2830, "optime" : { "t" : 1358380658000, "i" : 1813 }, "optimeDate" : ISODate("2013-01-16T23:57:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:41Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2840, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2827, "optime" : { "t" : 1358380658000, "i" : 1813 }, "optimeDate" : ISODate("2013-01-16T23:57:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2827, "optime" : { "t" : 1358380661000, "i" : 6142 }, "optimeDate" : ISODate("2013-01-16T23:57:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2827, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:41Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2839, "optime" : { "t" : 1358380662000, "i" : 3655 }, "optimeDate" : ISODate("2013-01-16T23:57:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:57:42.531 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:57:44.531 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:57:45.378 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:57:45.886 [conn170] end connection 127.0.0.2:50393 (3 connections now open) m31002| Wed Jan 16 18:57:46.386 [initandlisten] connection accepted from 127.0.0.2:59457 #172 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2845, "optime" : { "t" : 1358380666000, "i" : 4934 }, "optimeDate" : ISODate("2013-01-16T23:57:46Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2834, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2834, "optime" : { "t" : 1358380666000, "i" : 3938 }, "optimeDate" : ISODate("2013-01-16T23:57:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2844, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2831, "optime" : { "t" : 1358380662000, "i" : 3655 }, "optimeDate" : ISODate("2013-01-16T23:57:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:45Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2831, "optime" : { "t" : 1358380665000, "i" : 5026 }, "optimeDate" : ISODate("2013-01-16T23:57:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2831, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2843, "optime" : { "t" : 1358380666000, "i" : 4795 }, "optimeDate" : ISODate("2013-01-16T23:57:46Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:57:46.532 [conn169] end connection 127.0.0.2:49463 (2 connections now open) m31001| Wed Jan 16 18:57:46.532 [initandlisten] connection accepted from 127.0.0.2:48154 #170 (3 connections now open) m31000| Wed Jan 16 18:57:46.532 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:57:48.533 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:57:49.979 [conn104] end connection 127.0.0.4:44952 (4 connections now open) m31000| Wed Jan 16 18:57:49.979 [initandlisten] connection accepted from 127.0.0.4:56921 #105 (5 connections now open) m31001| Wed Jan 16 18:57:50.379 [conn168] end connection 127.0.0.4:55349 (2 connections now open) m31000| Wed Jan 16 18:57:50.533 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2849, "optime" : { "t" : 1358380670000, "i" : 5687 }, "optimeDate" : ISODate("2013-01-16T23:57:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2838, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2838, "optime" : { "t" : 1358380666000, "i" : 4934 }, "optimeDate" : ISODate("2013-01-16T23:57:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:49Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2848, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2835, "optime" : { "t" : 1358380666000, "i" : 4934 }, "optimeDate" : ISODate("2013-01-16T23:57:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2835, "optime" : { "t" : 1358380669000, "i" : 4146 }, "optimeDate" : ISODate("2013-01-16T23:57:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2835, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2847, "optime" : { "t" : 1358380670000, "i" : 5687 }, "optimeDate" : ISODate("2013-01-16T23:57:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:57:50.879 [initandlisten] connection accepted from 127.0.0.4:44588 #171 (3 connections now open) m31002| Wed Jan 16 18:57:50.879 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:57:51.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:57:52.533 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:57:54.534 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2853, "optime" : { "t" : 1358380674000, "i" : 6792 }, "optimeDate" : ISODate("2013-01-16T23:57:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2842, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2842, "optime" : { "t" : 1358380673000, "i" : 1978 }, "optimeDate" : ISODate("2013-01-16T23:57:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:53Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2852, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2839, "optime" : { "t" : 1358380670000, "i" : 5687 }, "optimeDate" : ISODate("2013-01-16T23:57:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:53Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2839, "optime" : { "t" : 1358380673000, "i" : 2933 }, "optimeDate" : ISODate("2013-01-16T23:57:53Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2839, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2851, "optime" : { "t" : 1358380674000, "i" : 6792 }, "optimeDate" : ISODate("2013-01-16T23:57:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:57:55.880 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:57:56.534 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:57:58.534 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2858, "optime" : { "t" : 1358380679000, "i" : 53 }, "optimeDate" : ISODate("2013-01-16T23:57:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2847, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2847, "optime" : { "t" : 1358380678000, "i" : 7138 }, "optimeDate" : ISODate("2013-01-16T23:57:58Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2857, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2844, "optime" : { "t" : 1358380674000, "i" : 6792 }, "optimeDate" : ISODate("2013-01-16T23:57:54Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:58Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:57:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2844, "optime" : { "t" : 1358380677000, "i" : 942 }, "optimeDate" : ISODate("2013-01-16T23:57:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:57:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2844, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2856, "optime" : { "t" : 1358380678000, "i" : 8346 }, "optimeDate" : ISODate("2013-01-16T23:57:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:58:00.535 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:58:00.881 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:58:01.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:58:02.535 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:58:03.134 [conn171] end connection 127.0.0.3:49866 (3 connections now open) m31002| Wed Jan 16 18:58:03.135 [initandlisten] connection accepted from 127.0.0.3:43160 #173 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2862, "optime" : { "t" : 1358380683000, "i" : 1537 }, "optimeDate" : ISODate("2013-01-16T23:58:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2851, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2851, "optime" : { "t" : 1358380679000, "i" : 53 }, "optimeDate" : ISODate("2013-01-16T23:57:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:01Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2861, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2848, "optime" : { "t" : 1358380683000, "i" : 1200 }, "optimeDate" : ISODate("2013-01-16T23:58:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2848, "optime" : { "t" : 1358380679000, "i" : 53 }, "optimeDate" : ISODate("2013-01-16T23:57:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2848, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:03Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2860, "optime" : { "t" : 1358380683000, "i" : 1537 }, "optimeDate" : ISODate("2013-01-16T23:58:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:58:04.535 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:58:05.881 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:58:06.536 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2866, "optime" : { "t" : 1358380687000, "i" : 2607 }, "optimeDate" : ISODate("2013-01-16T23:58:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2855, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2855, "optime" : { "t" : 1358380686000, "i" : 1680 }, "optimeDate" : ISODate("2013-01-16T23:58:06Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2865, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2852, "optime" : { "t" : 1358380687000, "i" : 1318 }, "optimeDate" : ISODate("2013-01-16T23:58:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2852, "optime" : { "t" : 1358380683000, "i" : 1537 }, "optimeDate" : ISODate("2013-01-16T23:58:03Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2852, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:07Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2864, "optime" : { "t" : 1358380687000, "i" : 2607 }, "optimeDate" : ISODate("2013-01-16T23:58:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:58:08.536 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:58:10.536 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:58:10.882 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2870, "optime" : { "t" : 1358380691000, "i" : 3941 }, "optimeDate" : ISODate("2013-01-16T23:58:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2859, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2859, "optime" : { "t" : 1358380691000, "i" : 3013 }, "optimeDate" : ISODate("2013-01-16T23:58:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:57:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2869, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2856, "optime" : { "t" : 1358380691000, "i" : 620 }, "optimeDate" : ISODate("2013-01-16T23:58:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2856, "optime" : { "t" : 1358380687000, "i" : 2607 }, "optimeDate" : ISODate("2013-01-16T23:58:07Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:11Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2856, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2868, "optime" : { "t" : 1358380691000, "i" : 3941 }, "optimeDate" : ISODate("2013-01-16T23:58:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:58:12.537 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:58:14.537 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2874, "optime" : { "t" : 1358380695000, "i" : 5539 }, "optimeDate" : ISODate("2013-01-16T23:58:15Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2863, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2863, "optime" : { "t" : 1358380691000, "i" : 3941 }, "optimeDate" : ISODate("2013-01-16T23:58:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:13Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2873, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2860, "optime" : { "t" : 1358380695000, "i" : 1153 }, "optimeDate" : ISODate("2013-01-16T23:58:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2860, "optime" : { "t" : 1358380691000, "i" : 3941 }, "optimeDate" : ISODate("2013-01-16T23:58:11Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2860, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:15Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2872, "optime" : { "t" : 1358380695000, "i" : 5539 }, "optimeDate" : ISODate("2013-01-16T23:58:15Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:58:15.883 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:58:16.537 [conn170] end connection 127.0.0.2:48154 (2 connections now open) m31001| Wed Jan 16 18:58:16.537 [initandlisten] connection accepted from 127.0.0.2:36382 #172 (3 connections now open) m31000| Wed Jan 16 18:58:16.538 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:58:18.538 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:58:18.847 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:418595 209ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2878, "optime" : { "t" : 1358380699000, "i" : 5956 }, "optimeDate" : ISODate("2013-01-16T23:58:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2867, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2867, "optime" : { "t" : 1358380698000, "i" : 945 }, "optimeDate" : ISODate("2013-01-16T23:58:18Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2877, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2864, "optime" : { "t" : 1358380699000, "i" : 874 }, "optimeDate" : ISODate("2013-01-16T23:58:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2864, "optime" : { "t" : 1358380695000, "i" : 5539 }, "optimeDate" : ISODate("2013-01-16T23:58:15Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2864, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:19Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2876, "optime" : { "t" : 1358380699000, "i" : 5330 }, "optimeDate" : ISODate("2013-01-16T23:58:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:58:19.985 [conn105] end connection 127.0.0.4:56921 (4 connections now open) m31000| Wed Jan 16 18:58:19.985 [initandlisten] connection accepted from 127.0.0.4:44658 #106 (5 connections now open) m31000| Wed Jan 16 18:58:20.538 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:58:20.883 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:58:22.539 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:58:23.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2882, "optime" : { "t" : 1358380703000, "i" : 6412 }, "optimeDate" : ISODate("2013-01-16T23:58:23Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2871, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2871, "optime" : { "t" : 1358380699000, "i" : 5956 }, "optimeDate" : ISODate("2013-01-16T23:58:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:21Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2881, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2868, "optime" : { "t" : 1358380703000, "i" : 1223 }, "optimeDate" : ISODate("2013-01-16T23:58:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:23Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2868, "optime" : { "t" : 1358380699000, "i" : 5956 }, "optimeDate" : ISODate("2013-01-16T23:58:19Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2868, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2880, "optime" : { "t" : 1358380703000, "i" : 6412 }, "optimeDate" : ISODate("2013-01-16T23:58:23Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:58:23.891 [conn172] end connection 127.0.0.2:59457 (3 connections now open) m31002| Wed Jan 16 18:58:24.391 [initandlisten] connection accepted from 127.0.0.2:36242 #174 (4 connections now open) m31000| Wed Jan 16 18:58:24.539 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:58:25.884 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:58:26.540 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2886, "optime" : { "t" : 1358380707000, "i" : 8109 }, "optimeDate" : ISODate("2013-01-16T23:58:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2875, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2875, "optime" : { "t" : 1358380706000, "i" : 1381 }, "optimeDate" : ISODate("2013-01-16T23:58:26Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2885, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2872, "optime" : { "t" : 1358380707000, "i" : 1029 }, "optimeDate" : ISODate("2013-01-16T23:58:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2872, "optime" : { "t" : 1358380703000, "i" : 6412 }, "optimeDate" : ISODate("2013-01-16T23:58:23Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2872, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:27Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2884, "optime" : { "t" : 1358380707000, "i" : 8109 }, "optimeDate" : ISODate("2013-01-16T23:58:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:58:28.384 [conn171] end connection 127.0.0.4:44588 (2 connections now open) m31000| Wed Jan 16 18:58:28.540 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:58:28.884 [initandlisten] connection accepted from 127.0.0.4:41176 #173 (3 connections now open) m31000| Wed Jan 16 18:58:30.541 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:58:31.385 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2890, "optime" : { "t" : 1358380711000, "i" : 8908 }, "optimeDate" : ISODate("2013-01-16T23:58:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2879, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2879, "optime" : { "t" : 1358380711000, "i" : 7528 }, "optimeDate" : ISODate("2013-01-16T23:58:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2889, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2876, "optime" : { "t" : 1358380711000, "i" : 1060 }, "optimeDate" : ISODate("2013-01-16T23:58:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:31Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2876, "optime" : { "t" : 1358380707000, "i" : 8109 }, "optimeDate" : ISODate("2013-01-16T23:58:27Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:31Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2876, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2888, "optime" : { "t" : 1358380711000, "i" : 8908 }, "optimeDate" : ISODate("2013-01-16T23:58:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:58:32.541 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:58:33.139 [conn173] end connection 127.0.0.3:43160 (3 connections now open) m31002| Wed Jan 16 18:58:33.139 [initandlisten] connection accepted from 127.0.0.3:55224 #175 (4 connections now open) m31001| Wed Jan 16 18:58:33.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:58:34.541 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2895, "optime" : { "t" : 1358380716000, "i" : 1173 }, "optimeDate" : ISODate("2013-01-16T23:58:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2884, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2884, "optime" : { "t" : 1358380711000, "i" : 8908 }, "optimeDate" : ISODate("2013-01-16T23:58:31Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:35Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2894, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2881, "optime" : { "t" : 1358380715000, "i" : 1411 }, "optimeDate" : ISODate("2013-01-16T23:58:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2881, "optime" : { "t" : 1358380715000, "i" : 8474 }, "optimeDate" : ISODate("2013-01-16T23:58:35Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2881, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:35Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2893, "optime" : { "t" : 1358380716000, "i" : 1173 }, "optimeDate" : ISODate("2013-01-16T23:58:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:58:36.386 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:58:36.542 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:58:38.542 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2899, "optime" : { "t" : 1358380720000, "i" : 2530 }, "optimeDate" : ISODate("2013-01-16T23:58:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2888, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2888, "optime" : { "t" : 1358380719000, "i" : 1799 }, "optimeDate" : ISODate("2013-01-16T23:58:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:39Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2898, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2885, "optime" : { "t" : 1358380716000, "i" : 1173 }, "optimeDate" : ISODate("2013-01-16T23:58:36Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2885, "optime" : { "t" : 1358380719000, "i" : 7432 }, "optimeDate" : ISODate("2013-01-16T23:58:39Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2885, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:39Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2897, "optime" : { "t" : 1358380720000, "i" : 2530 }, "optimeDate" : ISODate("2013-01-16T23:58:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:58:40.542 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:58:41.386 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:58:42.543 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:58:43.646 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:210918 105ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2903, "optime" : { "t" : 1358380724000, "i" : 3347 }, "optimeDate" : ISODate("2013-01-16T23:58:44Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2892, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2892, "optime" : { "t" : 1358380720000, "i" : 2530 }, "optimeDate" : ISODate("2013-01-16T23:58:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:43Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2902, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2889, "optime" : { "t" : 1358380720000, "i" : 2530 }, "optimeDate" : ISODate("2013-01-16T23:58:40Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:43Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2889, "optime" : { "t" : 1358380723000, "i" : 6181 }, "optimeDate" : ISODate("2013-01-16T23:58:43Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2889, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2901, "optime" : { "t" : 1358380724000, "i" : 3339 }, "optimeDate" : ISODate("2013-01-16T23:58:44Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:58:44.543 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:58:46.387 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:58:46.544 [conn172] end connection 127.0.0.2:36382 (2 connections now open) m31001| Wed Jan 16 18:58:46.544 [initandlisten] connection accepted from 127.0.0.2:43190 #174 (3 connections now open) m31000| Wed Jan 16 18:58:46.544 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:58:48.544 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 18:58:48.732 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:493797 246ms { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2907, "optime" : { "t" : 1358380728000, "i" : 4728 }, "optimeDate" : ISODate("2013-01-16T23:58:48Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2896, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2896, "optime" : { "t" : 1358380724000, "i" : 3347 }, "optimeDate" : ISODate("2013-01-16T23:58:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:47Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2906, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2893, "optime" : { "t" : 1358380724000, "i" : 3347 }, "optimeDate" : ISODate("2013-01-16T23:58:44Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2893, "optime" : { "t" : 1358380727000, "i" : 5177 }, "optimeDate" : ISODate("2013-01-16T23:58:47Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2893, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2905, "optime" : { "t" : 1358380728000, "i" : 3992 }, "optimeDate" : ISODate("2013-01-16T23:58:48Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:58:49.990 [conn106] end connection 127.0.0.4:44658 (4 connections now open) m31000| Wed Jan 16 18:58:49.990 [initandlisten] connection accepted from 127.0.0.4:43927 #107 (5 connections now open) m31000| Wed Jan 16 18:58:50.545 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:58:51.388 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:58:52.545 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2911, "optime" : { "t" : 1358380732000, "i" : 8118 }, "optimeDate" : ISODate("2013-01-16T23:58:52Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2900, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2900, "optime" : { "t" : 1358380731000, "i" : 1260 }, "optimeDate" : ISODate("2013-01-16T23:58:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:51Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2910, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2897, "optime" : { "t" : 1358380728000, "i" : 4728 }, "optimeDate" : ISODate("2013-01-16T23:58:48Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:51Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2897, "optime" : { "t" : 1358380731000, "i" : 1497 }, "optimeDate" : ISODate("2013-01-16T23:58:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2897, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2909, "optime" : { "t" : 1358380732000, "i" : 8118 }, "optimeDate" : ISODate("2013-01-16T23:58:52Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:58:54.545 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:58:55.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 18:58:56.388 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:58:56.546 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2916, "optime" : { "t" : 1358380737000, "i" : 398 }, "optimeDate" : ISODate("2013-01-16T23:58:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2905, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2905, "optime" : { "t" : 1358380736000, "i" : 8405 }, "optimeDate" : ISODate("2013-01-16T23:58:56Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2915, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2902, "optime" : { "t" : 1358380732000, "i" : 8118 }, "optimeDate" : ISODate("2013-01-16T23:58:52Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:56Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:58:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2902, "optime" : { "t" : 1358380735000, "i" : 382 }, "optimeDate" : ISODate("2013-01-16T23:58:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2902, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2914, "optime" : { "t" : 1358380737000, "i" : 398 }, "optimeDate" : ISODate("2013-01-16T23:58:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:58:58.546 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:59:00.546 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2920, "optime" : { "t" : 1358380741000, "i" : 1258 }, "optimeDate" : ISODate("2013-01-16T23:59:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2909, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2909, "optime" : { "t" : 1358380737000, "i" : 398 }, "optimeDate" : ISODate("2013-01-16T23:58:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:59Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2919, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2906, "optime" : { "t" : 1358380737000, "i" : 398 }, "optimeDate" : ISODate("2013-01-16T23:58:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2906, "optime" : { "t" : 1358380737000, "i" : 398 }, "optimeDate" : ISODate("2013-01-16T23:58:57Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2906, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:58:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2918, "optime" : { "t" : 1358380741000, "i" : 693 }, "optimeDate" : ISODate("2013-01-16T23:59:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:59:01.389 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 18:59:01.896 [conn174] end connection 127.0.0.2:36242 (3 connections now open) m31002| Wed Jan 16 18:59:02.397 [initandlisten] connection accepted from 127.0.0.2:45899 #176 (4 connections now open) m31000| Wed Jan 16 18:59:02.547 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:59:03.145 [conn175] end connection 127.0.0.3:55224 (3 connections now open) m31002| Wed Jan 16 18:59:03.145 [initandlisten] connection accepted from 127.0.0.3:55951 #177 (4 connections now open) m31000| Wed Jan 16 18:59:04.547 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2924, "optime" : { "t" : 1358380745000, "i" : 2627 }, "optimeDate" : ISODate("2013-01-16T23:59:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2913, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2913, "optime" : { "t" : 1358380744000, "i" : 5881 }, "optimeDate" : ISODate("2013-01-16T23:59:04Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2923, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2910, "optime" : { "t" : 1358380745000, "i" : 1183 }, "optimeDate" : ISODate("2013-01-16T23:59:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2910, "optime" : { "t" : 1358380741000, "i" : 1258 }, "optimeDate" : ISODate("2013-01-16T23:59:01Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2910, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:05Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2922, "optime" : { "t" : 1358380745000, "i" : 2545 }, "optimeDate" : ISODate("2013-01-16T23:59:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:59:05.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 18:59:06.389 [conn173] end connection 127.0.0.4:41176 (2 connections now open) m31000| Wed Jan 16 18:59:06.547 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:59:06.890 [initandlisten] connection accepted from 127.0.0.4:49643 #175 (3 connections now open) m31002| Wed Jan 16 18:59:06.890 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:59:08.548 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2928, "optime" : { "t" : 1358380749000, "i" : 3790 }, "optimeDate" : ISODate("2013-01-16T23:59:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2917, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2917, "optime" : { "t" : 1358380745000, "i" : 2627 }, "optimeDate" : ISODate("2013-01-16T23:59:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:07Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2927, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2914, "optime" : { "t" : 1358380749000, "i" : 1180 }, "optimeDate" : ISODate("2013-01-16T23:59:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:09Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2914, "optime" : { "t" : 1358380745000, "i" : 2627 }, "optimeDate" : ISODate("2013-01-16T23:59:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2914, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2926, "optime" : { "t" : 1358380749000, "i" : 3790 }, "optimeDate" : ISODate("2013-01-16T23:59:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:59:10.548 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:59:11.891 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:59:12.548 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2932, "optime" : { "t" : 1358380753000, "i" : 5363 }, "optimeDate" : ISODate("2013-01-16T23:59:13Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2921, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2921, "optime" : { "t" : 1358380749000, "i" : 3790 }, "optimeDate" : ISODate("2013-01-16T23:59:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:58:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2931, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2918, "optime" : { "t" : 1358380753000, "i" : 952 }, "optimeDate" : ISODate("2013-01-16T23:59:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2918, "optime" : { "t" : 1358380749000, "i" : 3790 }, "optimeDate" : ISODate("2013-01-16T23:59:09Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2918, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:13Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2930, "optime" : { "t" : 1358380753000, "i" : 5363 }, "optimeDate" : ISODate("2013-01-16T23:59:13Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:59:14.549 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:59:16.549 [conn174] end connection 127.0.0.2:43190 (2 connections now open) m31001| Wed Jan 16 18:59:16.549 [initandlisten] connection accepted from 127.0.0.2:41559 #176 (3 connections now open) m31000| Wed Jan 16 18:59:16.550 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:59:16.891 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2936, "optime" : { "t" : 1358380757000, "i" : 6485 }, "optimeDate" : ISODate("2013-01-16T23:59:17Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2925, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2925, "optime" : { "t" : 1358380757000, "i" : 3831 }, "optimeDate" : ISODate("2013-01-16T23:59:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2935, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2922, "optime" : { "t" : 1358380757000, "i" : 1377 }, "optimeDate" : ISODate("2013-01-16T23:59:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2922, "optime" : { "t" : 1358380753000, "i" : 5363 }, "optimeDate" : ISODate("2013-01-16T23:59:13Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:17Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2922, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2934, "optime" : { "t" : 1358380757000, "i" : 6485 }, "optimeDate" : ISODate("2013-01-16T23:59:17Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:59:18.550 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:59:19.996 [conn107] end connection 127.0.0.4:43927 (4 connections now open) m31000| Wed Jan 16 18:59:19.996 [initandlisten] connection accepted from 127.0.0.4:34544 #108 (5 connections now open) m31000| Wed Jan 16 18:59:20.550 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2940, "optime" : { "t" : 1358380761000, "i" : 7958 }, "optimeDate" : ISODate("2013-01-16T23:59:21Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2929, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2929, "optime" : { "t" : 1358380757000, "i" : 6485 }, "optimeDate" : ISODate("2013-01-16T23:59:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:19Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2939, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2926, "optime" : { "t" : 1358380761000, "i" : 1377 }, "optimeDate" : ISODate("2013-01-16T23:59:21Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2926, "optime" : { "t" : 1358380757000, "i" : 6485 }, "optimeDate" : ISODate("2013-01-16T23:59:17Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2926, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:21Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2938, "optime" : { "t" : 1358380761000, "i" : 7958 }, "optimeDate" : ISODate("2013-01-16T23:59:21Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:59:21.892 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:59:22.551 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:59:24.551 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2945, "optime" : { "t" : 1358380766000, "i" : 191 }, "optimeDate" : ISODate("2013-01-16T23:59:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2934, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2934, "optime" : { "t" : 1358380764000, "i" : 176 }, "optimeDate" : ISODate("2013-01-16T23:59:24Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:25Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2944, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2931, "optime" : { "t" : 1358380765000, "i" : 1182 }, "optimeDate" : ISODate("2013-01-16T23:59:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2931, "optime" : { "t" : 1358380765000, "i" : 8421 }, "optimeDate" : ISODate("2013-01-16T23:59:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2931, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2943, "optime" : { "t" : 1358380765000, "i" : 8421 }, "optimeDate" : ISODate("2013-01-16T23:59:25Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:59:26.551 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:59:26.893 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:59:27.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 18:59:28.552 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2949, "optime" : { "t" : 1358380770000, "i" : 1925 }, "optimeDate" : ISODate("2013-01-16T23:59:30Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2938, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2938, "optime" : { "t" : 1358380769000, "i" : 7236 }, "optimeDate" : ISODate("2013-01-16T23:59:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:29Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2948, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2935, "optime" : { "t" : 1358380769000, "i" : 557 }, "optimeDate" : ISODate("2013-01-16T23:59:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:29Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2935, "optime" : { "t" : 1358380769000, "i" : 7968 }, "optimeDate" : ISODate("2013-01-16T23:59:29Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2935, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2947, "optime" : { "t" : 1358380770000, "i" : 1925 }, "optimeDate" : ISODate("2013-01-16T23:59:30Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:59:30.552 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:59:31.893 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:59:32.553 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:59:33.150 [conn177] end connection 127.0.0.3:55951 (3 connections now open) m31002| Wed Jan 16 18:59:33.150 [initandlisten] connection accepted from 127.0.0.3:48681 #178 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2953, "optime" : { "t" : 1358380774000, "i" : 3118 }, "optimeDate" : ISODate("2013-01-16T23:59:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2942, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2942, "optime" : { "t" : 1358380770000, "i" : 1925 }, "optimeDate" : ISODate("2013-01-16T23:59:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:33Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2952, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2939, "optime" : { "t" : 1358380770000, "i" : 1925 }, "optimeDate" : ISODate("2013-01-16T23:59:30Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:34Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2939, "optime" : { "t" : 1358380773000, "i" : 6849 }, "optimeDate" : ISODate("2013-01-16T23:59:33Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2939, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2951, "optime" : { "t" : 1358380774000, "i" : 3118 }, "optimeDate" : ISODate("2013-01-16T23:59:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:59:34.553 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:59:36.553 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:59:36.894 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 18:59:37.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2957, "optime" : { "t" : 1358380778000, "i" : 4687 }, "optimeDate" : ISODate("2013-01-16T23:59:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2946, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2946, "optime" : { "t" : 1358380774000, "i" : 3118 }, "optimeDate" : ISODate("2013-01-16T23:59:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:37Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2956, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2943, "optime" : { "t" : 1358380774000, "i" : 3118 }, "optimeDate" : ISODate("2013-01-16T23:59:34Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2943, "optime" : { "t" : 1358380777000, "i" : 5285 }, "optimeDate" : ISODate("2013-01-16T23:59:37Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2943, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2955, "optime" : { "t" : 1358380778000, "i" : 4687 }, "optimeDate" : ISODate("2013-01-16T23:59:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:59:38.554 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:59:39.902 [conn176] end connection 127.0.0.2:45899 (3 connections now open) m31002| Wed Jan 16 18:59:40.402 [initandlisten] connection accepted from 127.0.0.2:41406 #179 (4 connections now open) m31000| Wed Jan 16 18:59:40.554 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:59:41.895 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:59:42.554 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2961, "optime" : { "t" : 1358380782000, "i" : 5774 }, "optimeDate" : ISODate("2013-01-16T23:59:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2950, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2950, "optime" : { "t" : 1358380778000, "i" : 4687 }, "optimeDate" : ISODate("2013-01-16T23:59:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:41Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2960, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2947, "optime" : { "t" : 1358380778000, "i" : 4687 }, "optimeDate" : ISODate("2013-01-16T23:59:38Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:41Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2947, "optime" : { "t" : 1358380781000, "i" : 4118 }, "optimeDate" : ISODate("2013-01-16T23:59:41Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2947, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2959, "optime" : { "t" : 1358380782000, "i" : 5725 }, "optimeDate" : ISODate("2013-01-16T23:59:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:59:44.395 [conn175] end connection 127.0.0.4:49643 (2 connections now open) m31000| Wed Jan 16 18:59:44.555 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 18:59:44.895 [initandlisten] connection accepted from 127.0.0.4:56423 #177 (3 connections now open) m31001| Wed Jan 16 18:59:46.555 [conn176] end connection 127.0.0.2:41559 (2 connections now open) m31001| Wed Jan 16 18:59:46.555 [initandlisten] connection accepted from 127.0.0.2:55622 #178 (3 connections now open) m31000| Wed Jan 16 18:59:46.555 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2965, "optime" : { "t" : 1358380786000, "i" : 7412 }, "optimeDate" : ISODate("2013-01-16T23:59:46Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2954, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2954, "optime" : { "t" : 1358380782000, "i" : 5774 }, "optimeDate" : ISODate("2013-01-16T23:59:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:45Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2964, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2951, "optime" : { "t" : 1358380782000, "i" : 5774 }, "optimeDate" : ISODate("2013-01-16T23:59:42Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2951, "optime" : { "t" : 1358380785000, "i" : 2571 }, "optimeDate" : ISODate("2013-01-16T23:59:45Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2951, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2963, "optime" : { "t" : 1358380786000, "i" : 7281 }, "optimeDate" : ISODate("2013-01-16T23:59:46Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:59:47.396 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:59:48.556 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:59:50.001 [conn108] end connection 127.0.0.4:34544 (4 connections now open) m31000| Wed Jan 16 18:59:50.001 [initandlisten] connection accepted from 127.0.0.4:39073 #109 (5 connections now open) m31000| Wed Jan 16 18:59:50.556 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2970, "optime" : { "t" : 1358380791000, "i" : 803 }, "optimeDate" : ISODate("2013-01-16T23:59:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2959, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2959, "optime" : { "t" : 1358380790000, "i" : 3983 }, "optimeDate" : ISODate("2013-01-16T23:59:50Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2969, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2956, "optime" : { "t" : 1358380786000, "i" : 7412 }, "optimeDate" : ISODate("2013-01-16T23:59:46Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:49Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2956, "optime" : { "t" : 1358380789000, "i" : 323 }, "optimeDate" : ISODate("2013-01-16T23:59:49Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2956, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2968, "optime" : { "t" : 1358380791000, "i" : 803 }, "optimeDate" : ISODate("2013-01-16T23:59:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 18:59:52.397 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:59:52.556 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:59:54.557 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2974, "optime" : { "t" : 1358380795000, "i" : 2263 }, "optimeDate" : ISODate("2013-01-16T23:59:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2963, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2963, "optime" : { "t" : 1358380791000, "i" : 803 }, "optimeDate" : ISODate("2013-01-16T23:59:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:53Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2973, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2960, "optime" : { "t" : 1358380795000, "i" : 732 }, "optimeDate" : ISODate("2013-01-16T23:59:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2960, "optime" : { "t" : 1358380791000, "i" : 803 }, "optimeDate" : ISODate("2013-01-16T23:59:51Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2960, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2972, "optime" : { "t" : 1358380795000, "i" : 1876 }, "optimeDate" : ISODate("2013-01-16T23:59:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 18:59:56.557 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 18:59:57.397 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 18:59:58.557 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 18:59:58.647 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:263042 131ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2978, "optime" : { "t" : 1358380799000, "i" : 3163 }, "optimeDate" : ISODate("2013-01-16T23:59:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2967, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2967, "optime" : { "t" : 1358380795000, "i" : 2263 }, "optimeDate" : ISODate("2013-01-16T23:59:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:57Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2977, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2964, "optime" : { "t" : 1358380799000, "i" : 1254 }, "optimeDate" : ISODate("2013-01-16T23:59:59Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-16T23:59:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2964, "optime" : { "t" : 1358380795000, "i" : 2263 }, "optimeDate" : ISODate("2013-01-16T23:59:55Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2964, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-16T23:59:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2976, "optime" : { "t" : 1358380799000, "i" : 2986 }, "optimeDate" : ISODate("2013-01-16T23:59:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 18:59:59.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:00:00.558 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:00:02.004 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:00:02.558 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:00:03.156 [conn178] end connection 127.0.0.3:48681 (3 connections now open) m31002| Wed Jan 16 19:00:03.156 [initandlisten] connection accepted from 127.0.0.3:44422 #180 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2982, "optime" : { "t" : 1358380803000, "i" : 4522 }, "optimeDate" : ISODate("2013-01-17T00:00:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2971, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2971, "optime" : { "t" : 1358380802000, "i" : 4396 }, "optimeDate" : ISODate("2013-01-17T00:00:02Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2981, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2968, "optime" : { "t" : 1358380803000, "i" : 944 }, "optimeDate" : ISODate("2013-01-17T00:00:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2968, "optime" : { "t" : 1358380799000, "i" : 3163 }, "optimeDate" : ISODate("2013-01-16T23:59:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2968, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:03Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2980, "optime" : { "t" : 1358380803000, "i" : 4522 }, "optimeDate" : ISODate("2013-01-17T00:00:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:00:04.558 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:00:06.559 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:00:07.398 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2986, "optime" : { "t" : 1358380807000, "i" : 6062 }, "optimeDate" : ISODate("2013-01-17T00:00:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2975, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2975, "optime" : { "t" : 1358380803000, "i" : 4522 }, "optimeDate" : ISODate("2013-01-17T00:00:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:06Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2985, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2972, "optime" : { "t" : 1358380807000, "i" : 1167 }, "optimeDate" : ISODate("2013-01-17T00:00:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:07Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2972, "optime" : { "t" : 1358380803000, "i" : 4522 }, "optimeDate" : ISODate("2013-01-17T00:00:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2972, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2984, "optime" : { "t" : 1358380807000, "i" : 6062 }, "optimeDate" : ISODate("2013-01-17T00:00:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:00:08.559 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:00:09.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:00:10.560 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2990, "optime" : { "t" : 1358380811000, "i" : 7857 }, "optimeDate" : ISODate("2013-01-17T00:00:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2979, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2979, "optime" : { "t" : 1358380807000, "i" : 6062 }, "optimeDate" : ISODate("2013-01-17T00:00:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2989, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2976, "optime" : { "t" : 1358380811000, "i" : 883 }, "optimeDate" : ISODate("2013-01-17T00:00:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2976, "optime" : { "t" : 1358380807000, "i" : 6062 }, "optimeDate" : ISODate("2013-01-17T00:00:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2976, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2988, "optime" : { "t" : 1358380811000, "i" : 7539 }, "optimeDate" : ISODate("2013-01-17T00:00:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:00:12.005 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:00:12.560 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:00:14.560 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2995, "optime" : { "t" : 1358380816000, "i" : 170 }, "optimeDate" : ISODate("2013-01-17T00:00:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2984, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2984, "optime" : { "t" : 1358380815000, "i" : 3253 }, "optimeDate" : ISODate("2013-01-17T00:00:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-16T23:59:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2994, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2981, "optime" : { "t" : 1358380815000, "i" : 1427 }, "optimeDate" : ISODate("2013-01-17T00:00:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2981, "optime" : { "t" : 1358380816000, "i" : 23 }, "optimeDate" : ISODate("2013-01-17T00:00:16Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2981, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:15Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2993, "optime" : { "t" : 1358380816000, "i" : 106 }, "optimeDate" : ISODate("2013-01-17T00:00:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:00:16.561 [conn178] end connection 127.0.0.2:55622 (2 connections now open) m31001| Wed Jan 16 19:00:16.561 [initandlisten] connection accepted from 127.0.0.2:44734 #179 (3 connections now open) m31000| Wed Jan 16 19:00:16.561 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:00:17.400 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 19:00:17.907 [conn179] end connection 127.0.0.2:41406 (3 connections now open) m31002| Wed Jan 16 19:00:18.407 [initandlisten] connection accepted from 127.0.0.2:42344 #181 (4 connections now open) m31000| Wed Jan 16 19:00:18.561 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:00:20.006 [conn109] end connection 127.0.0.4:39073 (4 connections now open) m31000| Wed Jan 16 19:00:20.006 [initandlisten] connection accepted from 127.0.0.4:59550 #110 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2999, "optime" : { "t" : 1358380820000, "i" : 2141 }, "optimeDate" : ISODate("2013-01-17T00:00:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2988, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2988, "optime" : { "t" : 1358380816000, "i" : 170 }, "optimeDate" : ISODate("2013-01-17T00:00:16Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2998, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2985, "optime" : { "t" : 1358380819000, "i" : 265 }, "optimeDate" : ISODate("2013-01-17T00:00:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:19Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2985, "optime" : { "t" : 1358380820000, "i" : 71 }, "optimeDate" : ISODate("2013-01-17T00:00:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2985, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2997, "optime" : { "t" : 1358380820000, "i" : 2141 }, "optimeDate" : ISODate("2013-01-17T00:00:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:00:20.562 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:00:22.007 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:00:22.400 [conn177] end connection 127.0.0.4:56423 (2 connections now open) m31000| Wed Jan 16 19:00:22.562 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:00:22.900 [initandlisten] connection accepted from 127.0.0.4:56417 #180 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3003, "optime" : { "t" : 1358380824000, "i" : 2954 }, "optimeDate" : ISODate("2013-01-17T00:00:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2992, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2992, "optime" : { "t" : 1358380823000, "i" : 1685 }, "optimeDate" : ISODate("2013-01-17T00:00:23Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:24Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3002, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2989, "optime" : { "t" : 1358380820000, "i" : 2141 }, "optimeDate" : ISODate("2013-01-17T00:00:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2989, "optime" : { "t" : 1358380824000, "i" : 31 }, "optimeDate" : ISODate("2013-01-17T00:00:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2989, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3001, "optime" : { "t" : 1358380824000, "i" : 2954 }, "optimeDate" : ISODate("2013-01-17T00:00:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:00:24.563 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:00:26.563 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:00:27.901 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3007, "optime" : { "t" : 1358380828000, "i" : 4598 }, "optimeDate" : ISODate("2013-01-17T00:00:28Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2996, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2996, "optime" : { "t" : 1358380828000, "i" : 4134 }, "optimeDate" : ISODate("2013-01-17T00:00:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3006, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2993, "optime" : { "t" : 1358380824000, "i" : 2954 }, "optimeDate" : ISODate("2013-01-17T00:00:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:27Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:28Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2993, "optime" : { "t" : 1358380827000, "i" : 5401 }, "optimeDate" : ISODate("2013-01-17T00:00:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2993, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3005, "optime" : { "t" : 1358380828000, "i" : 4334 }, "optimeDate" : ISODate("2013-01-17T00:00:28Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:00:28.563 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:00:30.564 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:00:31.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:00:32.008 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:00:32.564 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3011, "optime" : { "t" : 1358380832000, "i" : 4926 }, "optimeDate" : ISODate("2013-01-17T00:00:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3000, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3000, "optime" : { "t" : 1358380828000, "i" : 4598 }, "optimeDate" : ISODate("2013-01-17T00:00:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:32Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3010, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 2997, "optime" : { "t" : 1358380828000, "i" : 4598 }, "optimeDate" : ISODate("2013-01-17T00:00:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2997, "optime" : { "t" : 1358380832000, "i" : 44 }, "optimeDate" : ISODate("2013-01-17T00:00:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 2997, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:31Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3009, "optime" : { "t" : 1358380832000, "i" : 4684 }, "optimeDate" : ISODate("2013-01-17T00:00:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:00:33.162 [conn180] end connection 127.0.0.3:44422 (3 connections now open) m31002| Wed Jan 16 19:00:33.162 [initandlisten] connection accepted from 127.0.0.3:48692 #182 (4 connections now open) m31000| Wed Jan 16 19:00:34.564 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:00:36.565 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3015, "optime" : { "t" : 1358380836000, "i" : 6776 }, "optimeDate" : ISODate("2013-01-17T00:00:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3004, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3004, "optime" : { "t" : 1358380835000, "i" : 2564 }, "optimeDate" : ISODate("2013-01-17T00:00:35Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:36Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3014, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3001, "optime" : { "t" : 1358380832000, "i" : 4926 }, "optimeDate" : ISODate("2013-01-17T00:00:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:35Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3001, "optime" : { "t" : 1358380836000, "i" : 129 }, "optimeDate" : ISODate("2013-01-17T00:00:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3001, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3013, "optime" : { "t" : 1358380836000, "i" : 6573 }, "optimeDate" : ISODate("2013-01-17T00:00:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:00:37.903 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:00:38.565 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:00:40.565 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3019, "optime" : { "t" : 1358380840000, "i" : 8277 }, "optimeDate" : ISODate("2013-01-17T00:00:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3008, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3008, "optime" : { "t" : 1358380840000, "i" : 7847 }, "optimeDate" : ISODate("2013-01-17T00:00:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3018, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3005, "optime" : { "t" : 1358380836000, "i" : 6776 }, "optimeDate" : ISODate("2013-01-17T00:00:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:40Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3005, "optime" : { "t" : 1358380839000, "i" : 1722 }, "optimeDate" : ISODate("2013-01-17T00:00:39Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3005, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3017, "optime" : { "t" : 1358380840000, "i" : 8277 }, "optimeDate" : ISODate("2013-01-17T00:00:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:00:41.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:00:42.010 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:00:42.566 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:00:44.566 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3024, "optime" : { "t" : 1358380845000, "i" : 839 }, "optimeDate" : ISODate("2013-01-17T00:00:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3013, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3013, "optime" : { "t" : 1358380840000, "i" : 8277 }, "optimeDate" : ISODate("2013-01-17T00:00:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:44Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3023, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3010, "optime" : { "t" : 1358380840000, "i" : 8277 }, "optimeDate" : ISODate("2013-01-17T00:00:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3010, "optime" : { "t" : 1358380844000, "i" : 53 }, "optimeDate" : ISODate("2013-01-17T00:00:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3010, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:43Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3022, "optime" : { "t" : 1358380845000, "i" : 765 }, "optimeDate" : ISODate("2013-01-17T00:00:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:00:46.566 [conn179] end connection 127.0.0.2:44734 (2 connections now open) m31001| Wed Jan 16 19:00:46.566 [initandlisten] connection accepted from 127.0.0.2:41957 #181 (3 connections now open) m31000| Wed Jan 16 19:00:46.567 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:00:47.904 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:00:48.567 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3028, "optime" : { "t" : 1358380849000, "i" : 2390 }, "optimeDate" : ISODate("2013-01-17T00:00:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3017, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3017, "optime" : { "t" : 1358380848000, "i" : 2465 }, "optimeDate" : ISODate("2013-01-17T00:00:48Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3027, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3014, "optime" : { "t" : 1358380849000, "i" : 1510 }, "optimeDate" : ISODate("2013-01-17T00:00:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3014, "optime" : { "t" : 1358380845000, "i" : 839 }, "optimeDate" : ISODate("2013-01-17T00:00:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3014, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3026, "optime" : { "t" : 1358380849000, "i" : 2159 }, "optimeDate" : ISODate("2013-01-17T00:00:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:00:50.011 [conn110] end connection 127.0.0.4:59550 (4 connections now open) m31000| Wed Jan 16 19:00:50.011 [initandlisten] connection accepted from 127.0.0.4:48648 #111 (5 connections now open) m31000| Wed Jan 16 19:00:50.567 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:00:52.012 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:00:52.568 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3032, "optime" : { "t" : 1358380853000, "i" : 3017 }, "optimeDate" : ISODate("2013-01-17T00:00:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3021, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3021, "optime" : { "t" : 1358380849000, "i" : 2390 }, "optimeDate" : ISODate("2013-01-17T00:00:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:52Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3031, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3018, "optime" : { "t" : 1358380853000, "i" : 287 }, "optimeDate" : ISODate("2013-01-17T00:00:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3018, "optime" : { "t" : 1358380849000, "i" : 2390 }, "optimeDate" : ISODate("2013-01-17T00:00:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3018, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:53Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3030, "optime" : { "t" : 1358380853000, "i" : 2739 }, "optimeDate" : ISODate("2013-01-17T00:00:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:00:54.568 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:00:55.912 [conn181] end connection 127.0.0.2:42344 (3 connections now open) m31002| Wed Jan 16 19:00:56.412 [initandlisten] connection accepted from 127.0.0.2:60835 #183 (4 connections now open) m31000| Wed Jan 16 19:00:56.568 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3036, "optime" : { "t" : 1358380857000, "i" : 4571 }, "optimeDate" : ISODate("2013-01-17T00:00:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3025, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3025, "optime" : { "t" : 1358380856000, "i" : 21 }, "optimeDate" : ISODate("2013-01-17T00:00:56Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3035, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3022, "optime" : { "t" : 1358380857000, "i" : 1396 }, "optimeDate" : ISODate("2013-01-17T00:00:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:00:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3022, "optime" : { "t" : 1358380853000, "i" : 3017 }, "optimeDate" : ISODate("2013-01-17T00:00:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3022, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:00:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3034, "optime" : { "t" : 1358380857000, "i" : 4481 }, "optimeDate" : ISODate("2013-01-17T00:00:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:00:57.905 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:00:58.569 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:01:00.405 [conn180] end connection 127.0.0.4:56417 (2 connections now open) m31000| Wed Jan 16 19:01:00.569 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:01:00.905 [initandlisten] connection accepted from 127.0.0.4:44570 #182 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3040, "optime" : { "t" : 1358380861000, "i" : 5583 }, "optimeDate" : ISODate("2013-01-17T00:01:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3029, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3029, "optime" : { "t" : 1358380861000, "i" : 2928 }, "optimeDate" : ISODate("2013-01-17T00:01:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3039, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3026, "optime" : { "t" : 1358380861000, "i" : 1366 }, "optimeDate" : ISODate("2013-01-17T00:01:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3026, "optime" : { "t" : 1358380857000, "i" : 4571 }, "optimeDate" : ISODate("2013-01-17T00:00:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:01Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3026, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3038, "optime" : { "t" : 1358380861000, "i" : 5583 }, "optimeDate" : ISODate("2013-01-17T00:01:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:01:02.014 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:01:02.570 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:01:03.167 [conn182] end connection 127.0.0.3:48692 (3 connections now open) m31002| Wed Jan 16 19:01:03.167 [initandlisten] connection accepted from 127.0.0.3:54128 #184 (4 connections now open) m31001| Wed Jan 16 19:01:03.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:01:04.570 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3044, "optime" : { "t" : 1358380865000, "i" : 6800 }, "optimeDate" : ISODate("2013-01-17T00:01:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3033, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3033, "optime" : { "t" : 1358380861000, "i" : 5583 }, "optimeDate" : ISODate("2013-01-17T00:01:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:04Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3043, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3030, "optime" : { "t" : 1358380865000, "i" : 1122 }, "optimeDate" : ISODate("2013-01-17T00:01:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3030, "optime" : { "t" : 1358380861000, "i" : 5583 }, "optimeDate" : ISODate("2013-01-17T00:01:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3030, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:05Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3042, "optime" : { "t" : 1358380865000, "i" : 6312 }, "optimeDate" : ISODate("2013-01-17T00:01:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:01:06.570 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:01:08.015 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:01:08.571 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3048, "optime" : { "t" : 1358380869000, "i" : 8186 }, "optimeDate" : ISODate("2013-01-17T00:01:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3037, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3037, "optime" : { "t" : 1358380868000, "i" : 389 }, "optimeDate" : ISODate("2013-01-17T00:01:08Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3047, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3034, "optime" : { "t" : 1358380869000, "i" : 1256 }, "optimeDate" : ISODate("2013-01-17T00:01:09Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3034, "optime" : { "t" : 1358380865000, "i" : 6800 }, "optimeDate" : ISODate("2013-01-17T00:01:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3034, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:09Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3046, "optime" : { "t" : 1358380869000, "i" : 7719 }, "optimeDate" : ISODate("2013-01-17T00:01:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:01:10.571 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:01:12.572 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:01:13.407 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:01:13.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:01:13.850 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:559730 279ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3053, "optime" : { "t" : 1358380874000, "i" : 1297 }, "optimeDate" : ISODate("2013-01-17T00:01:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3042, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3042, "optime" : { "t" : 1358380873000, "i" : 6433 }, "optimeDate" : ISODate("2013-01-17T00:01:13Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:14Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3052, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3039, "optime" : { "t" : 1358380873000, "i" : 1152 }, "optimeDate" : ISODate("2013-01-17T00:01:13Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:13Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3039, "optime" : { "t" : 1358380873000, "i" : 7534 }, "optimeDate" : ISODate("2013-01-17T00:01:13Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3039, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3051, "optime" : { "t" : 1358380874000, "i" : 1190 }, "optimeDate" : ISODate("2013-01-17T00:01:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:01:14.572 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:01:16.572 [conn181] end connection 127.0.0.2:41957 (2 connections now open) m31001| Wed Jan 16 19:01:16.572 [initandlisten] connection accepted from 127.0.0.2:48625 #183 (3 connections now open) m31000| Wed Jan 16 19:01:16.573 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:01:18.016 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3057, "optime" : { "t" : 1358380878000, "i" : 2359 }, "optimeDate" : ISODate("2013-01-17T00:01:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3046, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3046, "optime" : { "t" : 1358380874000, "i" : 1297 }, "optimeDate" : ISODate("2013-01-17T00:01:14Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:18Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3056, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3043, "optime" : { "t" : 1358380877000, "i" : 183 }, "optimeDate" : ISODate("2013-01-17T00:01:17Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3043, "optime" : { "t" : 1358380878000, "i" : 286 }, "optimeDate" : ISODate("2013-01-17T00:01:18Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3043, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3055, "optime" : { "t" : 1358380878000, "i" : 2260 }, "optimeDate" : ISODate("2013-01-17T00:01:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:01:18.573 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:01:20.017 [conn111] end connection 127.0.0.4:48648 (4 connections now open) m31000| Wed Jan 16 19:01:20.017 [initandlisten] connection accepted from 127.0.0.4:50717 #112 (5 connections now open) m31000| Wed Jan 16 19:01:20.574 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3061, "optime" : { "t" : 1358380882000, "i" : 3776 }, "optimeDate" : ISODate("2013-01-17T00:01:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3050, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3050, "optime" : { "t" : 1358380881000, "i" : 1078 }, "optimeDate" : ISODate("2013-01-17T00:01:21Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:22Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:00:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3060, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3047, "optime" : { "t" : 1358380878000, "i" : 2359 }, "optimeDate" : ISODate("2013-01-17T00:01:18Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3047, "optime" : { "t" : 1358380881000, "i" : 6224 }, "optimeDate" : ISODate("2013-01-17T00:01:21Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3047, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:21Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3059, "optime" : { "t" : 1358380882000, "i" : 3708 }, "optimeDate" : ISODate("2013-01-17T00:01:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:01:22.574 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:01:23.409 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:01:24.574 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3065, "optime" : { "t" : 1358380886000, "i" : 4278 }, "optimeDate" : ISODate("2013-01-17T00:01:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3054, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3054, "optime" : { "t" : 1358380886000, "i" : 3665 }, "optimeDate" : ISODate("2013-01-17T00:01:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3064, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3051, "optime" : { "t" : 1358380882000, "i" : 3776 }, "optimeDate" : ISODate("2013-01-17T00:01:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:25Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3051, "optime" : { "t" : 1358380886000, "i" : 58 }, "optimeDate" : ISODate("2013-01-17T00:01:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3051, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3063, "optime" : { "t" : 1358380886000, "i" : 3869 }, "optimeDate" : ISODate("2013-01-17T00:01:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:01:26.575 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:01:28.018 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:01:28.575 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:01:30.575 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3069, "optime" : { "t" : 1358380890000, "i" : 5472 }, "optimeDate" : ISODate("2013-01-17T00:01:30Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3058, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3058, "optime" : { "t" : 1358380886000, "i" : 4278 }, "optimeDate" : ISODate("2013-01-17T00:01:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:30Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3068, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3055, "optime" : { "t" : 1358380886000, "i" : 4278 }, "optimeDate" : ISODate("2013-01-17T00:01:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3055, "optime" : { "t" : 1358380890000, "i" : 65 }, "optimeDate" : ISODate("2013-01-17T00:01:30Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3055, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:29Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3067, "optime" : { "t" : 1358380890000, "i" : 5472 }, "optimeDate" : ISODate("2013-01-17T00:01:30Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:01:32.576 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:01:33.173 [conn184] end connection 127.0.0.3:54128 (3 connections now open) m31002| Wed Jan 16 19:01:33.173 [initandlisten] connection accepted from 127.0.0.3:56706 #185 (4 connections now open) m31002| Wed Jan 16 19:01:33.410 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 19:01:33.917 [conn183] end connection 127.0.0.2:60835 (3 connections now open) m31002| Wed Jan 16 19:01:34.417 [initandlisten] connection accepted from 127.0.0.2:42008 #186 (4 connections now open) m31000| Wed Jan 16 19:01:34.576 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3073, "optime" : { "t" : 1358380894000, "i" : 6727 }, "optimeDate" : ISODate("2013-01-17T00:01:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3062, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3062, "optime" : { "t" : 1358380894000, "i" : 4072 }, "optimeDate" : ISODate("2013-01-17T00:01:34Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3072, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3059, "optime" : { "t" : 1358380890000, "i" : 5472 }, "optimeDate" : ISODate("2013-01-17T00:01:30Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:33Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3059, "optime" : { "t" : 1358380893000, "i" : 3272 }, "optimeDate" : ISODate("2013-01-17T00:01:33Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3059, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3071, "optime" : { "t" : 1358380894000, "i" : 6463 }, "optimeDate" : ISODate("2013-01-17T00:01:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:01:35.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:01:36.577 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:01:38.020 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:01:38.411 [conn182] end connection 127.0.0.4:44570 (2 connections now open) m31000| Wed Jan 16 19:01:38.577 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3077, "optime" : { "t" : 1358380898000, "i" : 7289 }, "optimeDate" : ISODate("2013-01-17T00:01:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3066, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3066, "optime" : { "t" : 1358380894000, "i" : 6727 }, "optimeDate" : ISODate("2013-01-17T00:01:34Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:38Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3076, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3063, "optime" : { "t" : 1358380894000, "i" : 6727 }, "optimeDate" : ISODate("2013-01-17T00:01:34Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3063, "optime" : { "t" : 1358380898000, "i" : 124 }, "optimeDate" : ISODate("2013-01-17T00:01:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3063, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3075, "optime" : { "t" : 1358380898000, "i" : 7289 }, "optimeDate" : ISODate("2013-01-17T00:01:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:01:38.911 [initandlisten] connection accepted from 127.0.0.4:55206 #184 (3 connections now open) m31000| Wed Jan 16 19:01:40.577 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:01:42.578 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3081, "optime" : { "t" : 1358380902000, "i" : 8556 }, "optimeDate" : ISODate("2013-01-17T00:01:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3070, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3070, "optime" : { "t" : 1358380901000, "i" : 352 }, "optimeDate" : ISODate("2013-01-17T00:01:41Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:42Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3080, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3067, "optime" : { "t" : 1358380898000, "i" : 7289 }, "optimeDate" : ISODate("2013-01-17T00:01:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:41Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3067, "optime" : { "t" : 1358380902000, "i" : 186 }, "optimeDate" : ISODate("2013-01-17T00:01:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3067, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3079, "optime" : { "t" : 1358380902000, "i" : 8387 }, "optimeDate" : ISODate("2013-01-17T00:01:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:01:43.912 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:01:44.578 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:01:45.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 19:01:46.578 [conn183] end connection 127.0.0.2:48625 (2 connections now open) m31001| Wed Jan 16 19:01:46.578 [initandlisten] connection accepted from 127.0.0.2:48269 #185 (3 connections now open) m31000| Wed Jan 16 19:01:46.578 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3086, "optime" : { "t" : 1358380907000, "i" : 915 }, "optimeDate" : ISODate("2013-01-17T00:01:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3075, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3075, "optime" : { "t" : 1358380906000, "i" : 8461 }, "optimeDate" : ISODate("2013-01-17T00:01:46Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3085, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3072, "optime" : { "t" : 1358380902000, "i" : 8556 }, "optimeDate" : ISODate("2013-01-17T00:01:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:46Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3072, "optime" : { "t" : 1358380905000, "i" : 285 }, "optimeDate" : ISODate("2013-01-17T00:01:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3072, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3084, "optime" : { "t" : 1358380907000, "i" : 915 }, "optimeDate" : ISODate("2013-01-17T00:01:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:01:48.022 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:01:48.579 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:01:50.022 [conn112] end connection 127.0.0.4:50717 (4 connections now open) m31000| Wed Jan 16 19:01:50.022 [initandlisten] connection accepted from 127.0.0.4:53520 #113 (5 connections now open) m31000| Wed Jan 16 19:01:50.579 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3090, "optime" : { "t" : 1358380911000, "i" : 1427 }, "optimeDate" : ISODate("2013-01-17T00:01:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3079, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3079, "optime" : { "t" : 1358380907000, "i" : 915 }, "optimeDate" : ISODate("2013-01-17T00:01:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:50Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3089, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3076, "optime" : { "t" : 1358380907000, "i" : 915 }, "optimeDate" : ISODate("2013-01-17T00:01:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3076, "optime" : { "t" : 1358380907000, "i" : 915 }, "optimeDate" : ISODate("2013-01-17T00:01:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3076, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3088, "optime" : { "t" : 1358380911000, "i" : 1286 }, "optimeDate" : ISODate("2013-01-17T00:01:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:01:52.579 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:01:53.913 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:01:54.580 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3094, "optime" : { "t" : 1358380915000, "i" : 2660 }, "optimeDate" : ISODate("2013-01-17T00:01:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3083, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3083, "optime" : { "t" : 1358380914000, "i" : 1669 }, "optimeDate" : ISODate("2013-01-17T00:01:54Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3093, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3080, "optime" : { "t" : 1358380915000, "i" : 1601 }, "optimeDate" : ISODate("2013-01-17T00:01:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3080, "optime" : { "t" : 1358380911000, "i" : 1427 }, "optimeDate" : ISODate("2013-01-17T00:01:51Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3080, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3092, "optime" : { "t" : 1358380915000, "i" : 2660 }, "optimeDate" : ISODate("2013-01-17T00:01:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:01:56.580 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:01:58.024 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:01:58.580 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3098, "optime" : { "t" : 1358380919000, "i" : 3685 }, "optimeDate" : ISODate("2013-01-17T00:01:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3087, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3087, "optime" : { "t" : 1358380915000, "i" : 2660 }, "optimeDate" : ISODate("2013-01-17T00:01:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:58Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3097, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3084, "optime" : { "t" : 1358380919000, "i" : 1419 }, "optimeDate" : ISODate("2013-01-17T00:01:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:01:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3084, "optime" : { "t" : 1358380915000, "i" : 2660 }, "optimeDate" : ISODate("2013-01-17T00:01:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3084, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:01:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3096, "optime" : { "t" : 1358380919000, "i" : 3549 }, "optimeDate" : ISODate("2013-01-17T00:01:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:02:00.581 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:02:02.581 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:02:03.178 [conn185] end connection 127.0.0.3:56706 (3 connections now open) m31002| Wed Jan 16 19:02:03.179 [initandlisten] connection accepted from 127.0.0.3:36921 #187 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3102, "optime" : { "t" : 1358380923000, "i" : 4617 }, "optimeDate" : ISODate("2013-01-17T00:02:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3091, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3091, "optime" : { "t" : 1358380919000, "i" : 3685 }, "optimeDate" : ISODate("2013-01-17T00:01:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:02Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3101, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3088, "optime" : { "t" : 1358380923000, "i" : 1420 }, "optimeDate" : ISODate("2013-01-17T00:02:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3088, "optime" : { "t" : 1358380919000, "i" : 3685 }, "optimeDate" : ISODate("2013-01-17T00:01:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3088, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:03Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3100, "optime" : { "t" : 1358380923000, "i" : 4617 }, "optimeDate" : ISODate("2013-01-17T00:02:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:02:03.915 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:02:04.581 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:02:06.582 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:02:07.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3106, "optime" : { "t" : 1358380927000, "i" : 6942 }, "optimeDate" : ISODate("2013-01-17T00:02:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3095, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3095, "optime" : { "t" : 1358380926000, "i" : 2657 }, "optimeDate" : ISODate("2013-01-17T00:02:06Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3105, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3092, "optime" : { "t" : 1358380927000, "i" : 1156 }, "optimeDate" : ISODate("2013-01-17T00:02:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3092, "optime" : { "t" : 1358380923000, "i" : 4617 }, "optimeDate" : ISODate("2013-01-17T00:02:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3092, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:07Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3104, "optime" : { "t" : 1358380927000, "i" : 6450 }, "optimeDate" : ISODate("2013-01-17T00:02:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:02:08.026 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:02:08.582 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:02:10.583 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3110, "optime" : { "t" : 1358380931000, "i" : 8073 }, "optimeDate" : ISODate("2013-01-17T00:02:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3099, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3099, "optime" : { "t" : 1358380927000, "i" : 6942 }, "optimeDate" : ISODate("2013-01-17T00:02:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:10Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3109, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3096, "optime" : { "t" : 1358380931000, "i" : 1673 }, "optimeDate" : ISODate("2013-01-17T00:02:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:11Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3096, "optime" : { "t" : 1358380927000, "i" : 6942 }, "optimeDate" : ISODate("2013-01-17T00:02:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3096, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3108, "optime" : { "t" : 1358380931000, "i" : 8073 }, "optimeDate" : ISODate("2013-01-17T00:02:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:02:11.922 [conn186] end connection 127.0.0.2:42008 (3 connections now open) m31002| Wed Jan 16 19:02:12.422 [initandlisten] connection accepted from 127.0.0.2:57668 #188 (4 connections now open) m31000| Wed Jan 16 19:02:12.583 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:02:13.916 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:02:14.583 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3115, "optime" : { "t" : 1358380936000, "i" : 323 }, "optimeDate" : ISODate("2013-01-17T00:02:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3104, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3104, "optime" : { "t" : 1358380934000, "i" : 56 }, "optimeDate" : ISODate("2013-01-17T00:02:14Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3114, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3101, "optime" : { "t" : 1358380935000, "i" : 1281 }, "optimeDate" : ISODate("2013-01-17T00:02:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3101, "optime" : { "t" : 1358380936000, "i" : 101 }, "optimeDate" : ISODate("2013-01-17T00:02:16Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3101, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:15Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3113, "optime" : { "t" : 1358380936000, "i" : 101 }, "optimeDate" : ISODate("2013-01-17T00:02:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:02:16.416 [conn184] end connection 127.0.0.4:55206 (2 connections now open) m31001| Wed Jan 16 19:02:16.584 [conn185] end connection 127.0.0.2:48269 (1 connection now open) m31001| Wed Jan 16 19:02:16.584 [initandlisten] connection accepted from 127.0.0.2:41280 #186 (2 connections now open) m31000| Wed Jan 16 19:02:16.584 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:02:16.916 [initandlisten] connection accepted from 127.0.0.4:36031 #187 (3 connections now open) m31001| Wed Jan 16 19:02:17.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:02:18.027 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:02:18.584 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:02:20.028 [conn113] end connection 127.0.0.4:53520 (4 connections now open) m31000| Wed Jan 16 19:02:20.028 [initandlisten] connection accepted from 127.0.0.4:49942 #114 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3119, "optime" : { "t" : 1358380940000, "i" : 2034 }, "optimeDate" : ISODate("2013-01-17T00:02:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3108, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3108, "optime" : { "t" : 1358380939000, "i" : 7692 }, "optimeDate" : ISODate("2013-01-17T00:02:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3118, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3105, "optime" : { "t" : 1358380939000, "i" : 417 }, "optimeDate" : ISODate("2013-01-17T00:02:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:19Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3105, "optime" : { "t" : 1358380940000, "i" : 710 }, "optimeDate" : ISODate("2013-01-17T00:02:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3105, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3117, "optime" : { "t" : 1358380940000, "i" : 2034 }, "optimeDate" : ISODate("2013-01-17T00:02:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:02:20.585 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:02:22.585 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:02:23.923 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:810590 405ms m31002| Wed Jan 16 19:02:24.028 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3123, "optime" : { "t" : 1358380944000, "i" : 4880 }, "optimeDate" : ISODate("2013-01-17T00:02:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3112, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3112, "optime" : { "t" : 1358380940000, "i" : 2034 }, "optimeDate" : ISODate("2013-01-17T00:02:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:24Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:01:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3122, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3109, "optime" : { "t" : 1358380940000, "i" : 2034 }, "optimeDate" : ISODate("2013-01-17T00:02:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3109, "optime" : { "t" : 1358380944000, "i" : 790 }, "optimeDate" : ISODate("2013-01-17T00:02:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3109, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3121, "optime" : { "t" : 1358380944000, "i" : 3216 }, "optimeDate" : ISODate("2013-01-17T00:02:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:02:24.586 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:02:26.586 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3127, "optime" : { "t" : 1358380948000, "i" : 3663 }, "optimeDate" : ISODate("2013-01-17T00:02:28Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3116, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3116, "optime" : { "t" : 1358380947000, "i" : 995 }, "optimeDate" : ISODate("2013-01-17T00:02:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:28Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3126, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3113, "optime" : { "t" : 1358380944000, "i" : 4880 }, "optimeDate" : ISODate("2013-01-17T00:02:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:28Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3113, "optime" : { "t" : 1358380947000, "i" : 6337 }, "optimeDate" : ISODate("2013-01-17T00:02:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3113, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:27Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3125, "optime" : { "t" : 1358380948000, "i" : 3618 }, "optimeDate" : ISODate("2013-01-17T00:02:28Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:02:28.586 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:02:29.419 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:02:30.587 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3131, "optime" : { "t" : 1358380952000, "i" : 4764 }, "optimeDate" : ISODate("2013-01-17T00:02:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3120, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3120, "optime" : { "t" : 1358380952000, "i" : 3136 }, "optimeDate" : ISODate("2013-01-17T00:02:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3130, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3117, "optime" : { "t" : 1358380948000, "i" : 3663 }, "optimeDate" : ISODate("2013-01-17T00:02:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:31Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3117, "optime" : { "t" : 1358380952000, "i" : 194 }, "optimeDate" : ISODate("2013-01-17T00:02:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3117, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3129, "optime" : { "t" : 1358380952000, "i" : 4543 }, "optimeDate" : ISODate("2013-01-17T00:02:32Z"), "self" : true } ], "ok" : 1 } m31000| Wed Jan 16 19:02:32.587 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:02:33.184 [conn187] end connection 127.0.0.3:36921 (3 connections now open) m31002| Wed Jan 16 19:02:33.184 [initandlisten] connection accepted from 127.0.0.3:43398 #189 (4 connections now open) m31002| Wed Jan 16 19:02:34.030 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:02:34.588 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:02:36.588 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3135, "optime" : { "t" : 1358380956000, "i" : 6568 }, "optimeDate" : ISODate("2013-01-17T00:02:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3124, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3124, "optime" : { "t" : 1358380952000, "i" : 4764 }, "optimeDate" : ISODate("2013-01-17T00:02:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:36Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3134, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3121, "optime" : { "t" : 1358380952000, "i" : 4764 }, "optimeDate" : ISODate("2013-01-17T00:02:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3121, "optime" : { "t" : 1358380956000, "i" : 168 }, "optimeDate" : ISODate("2013-01-17T00:02:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3121, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:35Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3133, "optime" : { "t" : 1358380956000, "i" : 6568 }, "optimeDate" : ISODate("2013-01-17T00:02:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:02:38.588 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:02:39.420 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:02:39.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:02:40.589 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3139, "optime" : { "t" : 1358380960000, "i" : 7417 }, "optimeDate" : ISODate("2013-01-17T00:02:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3128, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3128, "optime" : { "t" : 1358380959000, "i" : 1425 }, "optimeDate" : ISODate("2013-01-17T00:02:39Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:40Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3138, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3125, "optime" : { "t" : 1358380956000, "i" : 6568 }, "optimeDate" : ISODate("2013-01-17T00:02:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:39Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3125, "optime" : { "t" : 1358380960000, "i" : 189 }, "optimeDate" : ISODate("2013-01-17T00:02:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3125, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3137, "optime" : { "t" : 1358380960000, "i" : 7417 }, "optimeDate" : ISODate("2013-01-17T00:02:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:02:42.589 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:02:44.031 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:02:44.589 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3144, "optime" : { "t" : 1358380965000, "i" : 148 }, "optimeDate" : ISODate("2013-01-17T00:02:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3133, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3133, "optime" : { "t" : 1358380964000, "i" : 7320 }, "optimeDate" : ISODate("2013-01-17T00:02:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3143, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3130, "optime" : { "t" : 1358380960000, "i" : 7417 }, "optimeDate" : ISODate("2013-01-17T00:02:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:44Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3130, "optime" : { "t" : 1358380964000, "i" : 207 }, "optimeDate" : ISODate("2013-01-17T00:02:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3130, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3142, "optime" : { "t" : 1358380965000, "i" : 139 }, "optimeDate" : ISODate("2013-01-17T00:02:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:02:46.590 [conn186] end connection 127.0.0.2:41280 (2 connections now open) m31001| Wed Jan 16 19:02:46.590 [initandlisten] connection accepted from 127.0.0.2:39227 #188 (3 connections now open) m31000| Wed Jan 16 19:02:46.590 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:02:48.590 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3148, "optime" : { "t" : 1358380969000, "i" : 1409 }, "optimeDate" : ISODate("2013-01-17T00:02:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3137, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3137, "optime" : { "t" : 1358380965000, "i" : 148 }, "optimeDate" : ISODate("2013-01-17T00:02:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:48Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3147, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3134, "optime" : { "t" : 1358380965000, "i" : 148 }, "optimeDate" : ISODate("2013-01-17T00:02:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3134, "optime" : { "t" : 1358380968000, "i" : 2 }, "optimeDate" : ISODate("2013-01-17T00:02:48Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3134, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3146, "optime" : { "t" : 1358380969000, "i" : 865 }, "optimeDate" : ISODate("2013-01-17T00:02:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:02:49.421 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:02:49.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:02:49.928 [conn188] end connection 127.0.0.2:57668 (3 connections now open) m31000| Wed Jan 16 19:02:50.032 [conn114] end connection 127.0.0.4:49942 (4 connections now open) m31000| Wed Jan 16 19:02:50.032 [initandlisten] connection accepted from 127.0.0.4:34221 #115 (5 connections now open) m31002| Wed Jan 16 19:02:50.428 [initandlisten] connection accepted from 127.0.0.2:43585 #190 (4 connections now open) m31000| Wed Jan 16 19:02:50.591 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:02:52.591 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:02:53.235 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:261105 130ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3152, "optime" : { "t" : 1358380973000, "i" : 3383 }, "optimeDate" : ISODate("2013-01-17T00:02:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3141, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3141, "optime" : { "t" : 1358380972000, "i" : 6271 }, "optimeDate" : ISODate("2013-01-17T00:02:52Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3151, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3138, "optime" : { "t" : 1358380973000, "i" : 888 }, "optimeDate" : ISODate("2013-01-17T00:02:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3138, "optime" : { "t" : 1358380969000, "i" : 1409 }, "optimeDate" : ISODate("2013-01-17T00:02:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3138, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:53Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3150, "optime" : { "t" : 1358380973000, "i" : 3383 }, "optimeDate" : ISODate("2013-01-17T00:02:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:02:54.033 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:02:54.422 [conn187] end connection 127.0.0.4:36031 (2 connections now open) m31000| Wed Jan 16 19:02:54.591 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:02:54.922 [initandlisten] connection accepted from 127.0.0.4:44241 #189 (3 connections now open) m31000| Wed Jan 16 19:02:56.592 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3156, "optime" : { "t" : 1358380977000, "i" : 4179 }, "optimeDate" : ISODate("2013-01-17T00:02:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3145, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3145, "optime" : { "t" : 1358380973000, "i" : 3383 }, "optimeDate" : ISODate("2013-01-17T00:02:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:56Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3155, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3142, "optime" : { "t" : 1358380977000, "i" : 1440 }, "optimeDate" : ISODate("2013-01-17T00:02:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:02:57Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:02:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3142, "optime" : { "t" : 1358380973000, "i" : 3383 }, "optimeDate" : ISODate("2013-01-17T00:02:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3142, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3154, "optime" : { "t" : 1358380977000, "i" : 4179 }, "optimeDate" : ISODate("2013-01-17T00:02:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:02:58.592 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:02:59.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:02:59.923 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:03:00.592 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3160, "optime" : { "t" : 1358380981000, "i" : 5712 }, "optimeDate" : ISODate("2013-01-17T00:03:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3149, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3149, "optime" : { "t" : 1358380977000, "i" : 4179 }, "optimeDate" : ISODate("2013-01-17T00:02:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3159, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3146, "optime" : { "t" : 1358380981000, "i" : 1547 }, "optimeDate" : ISODate("2013-01-17T00:03:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3146, "optime" : { "t" : 1358380977000, "i" : 4179 }, "optimeDate" : ISODate("2013-01-17T00:02:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3146, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3158, "optime" : { "t" : 1358380981000, "i" : 5712 }, "optimeDate" : ISODate("2013-01-17T00:03:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:03:02.593 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:03:03.189 [conn189] end connection 127.0.0.3:43398 (3 connections now open) m31002| Wed Jan 16 19:03:03.189 [initandlisten] connection accepted from 127.0.0.3:54098 #191 (4 connections now open) m31002| Wed Jan 16 19:03:04.035 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:03:04.593 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3164, "optime" : { "t" : 1358380985000, "i" : 6359 }, "optimeDate" : ISODate("2013-01-17T00:03:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3153, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3153, "optime" : { "t" : 1358380985000, "i" : 3114 }, "optimeDate" : ISODate("2013-01-17T00:03:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:02:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3163, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3150, "optime" : { "t" : 1358380985000, "i" : 1693 }, "optimeDate" : ISODate("2013-01-17T00:03:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3150, "optime" : { "t" : 1358380981000, "i" : 5712 }, "optimeDate" : ISODate("2013-01-17T00:03:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:05Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3150, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:05Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3162, "optime" : { "t" : 1358380985000, "i" : 5640 }, "optimeDate" : ISODate("2013-01-17T00:03:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:03:06.594 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:03:08.594 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 m31002| Wed Jan 16 19:03:09.924 [rsMgr] replSet info two primaries (transiently) { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3168, "optime" : { "t" : 1358380989000, "i" : 7560 }, "optimeDate" : ISODate("2013-01-17T00:03:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3157, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3157, "optime" : { "t" : 1358380985000, "i" : 6359 }, "optimeDate" : ISODate("2013-01-17T00:03:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:08Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3167, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3154, "optime" : { "t" : 1358380989000, "i" : 1199 }, "optimeDate" : ISODate("2013-01-17T00:03:09Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:09Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3154, "optime" : { "t" : 1358380985000, "i" : 6359 }, "optimeDate" : ISODate("2013-01-17T00:03:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3154, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3166, "optime" : { "t" : 1358380989000, "i" : 7560 }, "optimeDate" : ISODate("2013-01-17T00:03:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:03:10.595 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:03:12.595 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:03:14.037 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3173, "optime" : { "t" : 1358380994000, "i" : 892 }, "optimeDate" : ISODate("2013-01-17T00:03:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3162, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3162, "optime" : { "t" : 1358380989000, "i" : 7560 }, "optimeDate" : ISODate("2013-01-17T00:03:09Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:14Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3172, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3159, "optime" : { "t" : 1358380993000, "i" : 1578 }, "optimeDate" : ISODate("2013-01-17T00:03:13Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3159, "optime" : { "t" : 1358380994000, "i" : 247 }, "optimeDate" : ISODate("2013-01-17T00:03:14Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3159, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:13Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3171, "optime" : { "t" : 1358380994000, "i" : 392 }, "optimeDate" : ISODate("2013-01-17T00:03:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:03:14.595 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:03:16.596 [conn188] end connection 127.0.0.2:39227 (2 connections now open) m31001| Wed Jan 16 19:03:16.596 [initandlisten] connection accepted from 127.0.0.2:37472 #190 (3 connections now open) m31000| Wed Jan 16 19:03:16.596 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3177, "optime" : { "t" : 1358380998000, "i" : 2751 }, "optimeDate" : ISODate("2013-01-17T00:03:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3166, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3166, "optime" : { "t" : 1358380997000, "i" : 6992 }, "optimeDate" : ISODate("2013-01-17T00:03:17Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:18Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3176, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3163, "optime" : { "t" : 1358380997000, "i" : 69 }, "optimeDate" : ISODate("2013-01-17T00:03:17Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:17Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3163, "optime" : { "t" : 1358380998000, "i" : 340 }, "optimeDate" : ISODate("2013-01-17T00:03:18Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3163, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3175, "optime" : { "t" : 1358380998000, "i" : 2667 }, "optimeDate" : ISODate("2013-01-17T00:03:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:03:18.597 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:03:19.926 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:03:20.038 [conn115] end connection 127.0.0.4:34221 (4 connections now open) m31000| Wed Jan 16 19:03:20.038 [initandlisten] connection accepted from 127.0.0.4:38102 #116 (5 connections now open) m31000| Wed Jan 16 19:03:20.597 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:03:21.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3181, "optime" : { "t" : 1358381002000, "i" : 3905 }, "optimeDate" : ISODate("2013-01-17T00:03:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3170, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3170, "optime" : { "t" : 1358380998000, "i" : 2751 }, "optimeDate" : ISODate("2013-01-17T00:03:18Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:22Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3180, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3167, "optime" : { "t" : 1358380998000, "i" : 2751 }, "optimeDate" : ISODate("2013-01-17T00:03:18Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:22Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3167, "optime" : { "t" : 1358381001000, "i" : 6095 }, "optimeDate" : ISODate("2013-01-17T00:03:21Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3167, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3179, "optime" : { "t" : 1358381002000, "i" : 3905 }, "optimeDate" : ISODate("2013-01-17T00:03:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:03:22.597 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:03:24.039 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:03:24.598 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3185, "optime" : { "t" : 1358381006000, "i" : 4627 }, "optimeDate" : ISODate("2013-01-17T00:03:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3174, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3174, "optime" : { "t" : 1358381002000, "i" : 3905 }, "optimeDate" : ISODate("2013-01-17T00:03:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:26Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3184, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3171, "optime" : { "t" : 1358381002000, "i" : 3905 }, "optimeDate" : ISODate("2013-01-17T00:03:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3171, "optime" : { "t" : 1358381006000, "i" : 25 }, "optimeDate" : ISODate("2013-01-17T00:03:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3171, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3183, "optime" : { "t" : 1358381006000, "i" : 4603 }, "optimeDate" : ISODate("2013-01-17T00:03:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:03:26.598 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:03:27.933 [conn190] end connection 127.0.0.2:43585 (3 connections now open) m31002| Wed Jan 16 19:03:28.433 [initandlisten] connection accepted from 127.0.0.2:52848 #192 (4 connections now open) m31000| Wed Jan 16 19:03:28.599 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:03:29.927 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3189, "optime" : { "t" : 1358381010000, "i" : 5875 }, "optimeDate" : ISODate("2013-01-17T00:03:30Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3178, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3178, "optime" : { "t" : 1358381006000, "i" : 4627 }, "optimeDate" : ISODate("2013-01-17T00:03:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:30Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3188, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3175, "optime" : { "t" : 1358381006000, "i" : 4627 }, "optimeDate" : ISODate("2013-01-17T00:03:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:29Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3175, "optime" : { "t" : 1358381010000, "i" : 760 }, "optimeDate" : ISODate("2013-01-17T00:03:30Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3175, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3187, "optime" : { "t" : 1358381010000, "i" : 5398 }, "optimeDate" : ISODate("2013-01-17T00:03:30Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:03:30.599 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:03:31.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 19:03:32.427 [conn189] end connection 127.0.0.4:44241 (2 connections now open) m31000| Wed Jan 16 19:03:32.599 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:03:32.927 [initandlisten] connection accepted from 127.0.0.4:53921 #191 (3 connections now open) m31002| Wed Jan 16 19:03:33.194 [conn191] end connection 127.0.0.3:54098 (3 connections now open) m31002| Wed Jan 16 19:03:33.195 [initandlisten] connection accepted from 127.0.0.3:37117 #193 (4 connections now open) m31000| Wed Jan 16 19:03:34.004 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:772569 386ms m31002| Wed Jan 16 19:03:34.041 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3193, "optime" : { "t" : 1358381014000, "i" : 9864 }, "optimeDate" : ISODate("2013-01-17T00:03:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3182, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3182, "optime" : { "t" : 1358381010000, "i" : 5875 }, "optimeDate" : ISODate("2013-01-17T00:03:30Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:34Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3192, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3179, "optime" : { "t" : 1358381010000, "i" : 5875 }, "optimeDate" : ISODate("2013-01-17T00:03:30Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3179, "optime" : { "t" : 1358381014000, "i" : 772 }, "optimeDate" : ISODate("2013-01-17T00:03:34Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3179, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:33Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3191, "optime" : { "t" : 1358381014000, "i" : 9641 }, "optimeDate" : ISODate("2013-01-17T00:03:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:03:34.600 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:03:36.600 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:03:38.600 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 19:03:38.697 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:379120 189ms { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3197, "optime" : { "t" : 1358381018000, "i" : 5691 }, "optimeDate" : ISODate("2013-01-17T00:03:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3186, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3186, "optime" : { "t" : 1358381018000, "i" : 3912 }, "optimeDate" : ISODate("2013-01-17T00:03:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3196, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3183, "optime" : { "t" : 1358381014000, "i" : 9864 }, "optimeDate" : ISODate("2013-01-17T00:03:34Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:37Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3183, "optime" : { "t" : 1358381018000, "i" : 211 }, "optimeDate" : ISODate("2013-01-17T00:03:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3183, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3195, "optime" : { "t" : 1358381018000, "i" : 5691 }, "optimeDate" : ISODate("2013-01-17T00:03:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:03:40.042 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:03:40.601 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:03:42.601 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3201, "optime" : { "t" : 1358381022000, "i" : 7562 }, "optimeDate" : ISODate("2013-01-17T00:03:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3190, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3190, "optime" : { "t" : 1358381018000, "i" : 5691 }, "optimeDate" : ISODate("2013-01-17T00:03:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:42Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3200, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3187, "optime" : { "t" : 1358381018000, "i" : 5691 }, "optimeDate" : ISODate("2013-01-17T00:03:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:42Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3187, "optime" : { "t" : 1358381022000, "i" : 209 }, "optimeDate" : ISODate("2013-01-17T00:03:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3187, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3199, "optime" : { "t" : 1358381022000, "i" : 7562 }, "optimeDate" : ISODate("2013-01-17T00:03:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:03:44.602 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:03:45.429 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:03:46.602 [conn190] end connection 127.0.0.2:37472 (2 connections now open) m31001| Wed Jan 16 19:03:46.602 [initandlisten] connection accepted from 127.0.0.2:38023 #192 (3 connections now open) m31000| Wed Jan 16 19:03:46.602 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3206, "optime" : { "t" : 1358381027000, "i" : 906 }, "optimeDate" : ISODate("2013-01-17T00:03:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3195, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3195, "optime" : { "t" : 1358381022000, "i" : 7562 }, "optimeDate" : ISODate("2013-01-17T00:03:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:46Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3205, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3192, "optime" : { "t" : 1358381022000, "i" : 7562 }, "optimeDate" : ISODate("2013-01-17T00:03:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:45Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3192, "optime" : { "t" : 1358381026000, "i" : 126 }, "optimeDate" : ISODate("2013-01-17T00:03:46Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3192, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3204, "optime" : { "t" : 1358381027000, "i" : 389 }, "optimeDate" : ISODate("2013-01-17T00:03:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:03:48.602 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:03:50.043 [conn116] end connection 127.0.0.4:38102 (4 connections now open) m31000| Wed Jan 16 19:03:50.043 [initandlisten] connection accepted from 127.0.0.4:48025 #117 (5 connections now open) m31002| Wed Jan 16 19:03:50.044 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:03:50.603 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3210, "optime" : { "t" : 1358381031000, "i" : 1851 }, "optimeDate" : ISODate("2013-01-17T00:03:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3199, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3199, "optime" : { "t" : 1358381030000, "i" : 7317 }, "optimeDate" : ISODate("2013-01-17T00:03:50Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3209, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3196, "optime" : { "t" : 1358381031000, "i" : 1810 }, "optimeDate" : ISODate("2013-01-17T00:03:51Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3196, "optime" : { "t" : 1358381027000, "i" : 906 }, "optimeDate" : ISODate("2013-01-17T00:03:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3196, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3208, "optime" : { "t" : 1358381031000, "i" : 1851 }, "optimeDate" : ISODate("2013-01-17T00:03:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:03:52.603 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:03:53.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:03:54.603 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3214, "optime" : { "t" : 1358381035000, "i" : 2867 }, "optimeDate" : ISODate("2013-01-17T00:03:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3203, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3203, "optime" : { "t" : 1358381031000, "i" : 1851 }, "optimeDate" : ISODate("2013-01-17T00:03:51Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:54Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3213, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3200, "optime" : { "t" : 1358381035000, "i" : 1609 }, "optimeDate" : ISODate("2013-01-17T00:03:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3200, "optime" : { "t" : 1358381031000, "i" : 1851 }, "optimeDate" : ISODate("2013-01-17T00:03:51Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3200, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3212, "optime" : { "t" : 1358381035000, "i" : 2867 }, "optimeDate" : ISODate("2013-01-17T00:03:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:03:55.431 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:03:56.604 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:03:58.604 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3218, "optime" : { "t" : 1358381039000, "i" : 4563 }, "optimeDate" : ISODate("2013-01-17T00:03:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3207, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3207, "optime" : { "t" : 1358381038000, "i" : 486 }, "optimeDate" : ISODate("2013-01-17T00:03:58Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3217, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3204, "optime" : { "t" : 1358381039000, "i" : 1719 }, "optimeDate" : ISODate("2013-01-17T00:03:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:03:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3204, "optime" : { "t" : 1358381035000, "i" : 2867 }, "optimeDate" : ISODate("2013-01-17T00:03:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3204, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:03:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3216, "optime" : { "t" : 1358381039000, "i" : 4553 }, "optimeDate" : ISODate("2013-01-17T00:03:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:04:00.045 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:04:00.605 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:04:02.605 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:04:03.200 [conn193] end connection 127.0.0.3:37117 (3 connections now open) m31002| Wed Jan 16 19:04:03.200 [initandlisten] connection accepted from 127.0.0.3:54653 #194 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3222, "optime" : { "t" : 1358381043000, "i" : 5669 }, "optimeDate" : ISODate("2013-01-17T00:04:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3211, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3211, "optime" : { "t" : 1358381043000, "i" : 3966 }, "optimeDate" : ISODate("2013-01-17T00:04:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3221, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3208, "optime" : { "t" : 1358381043000, "i" : 2084 }, "optimeDate" : ISODate("2013-01-17T00:04:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3208, "optime" : { "t" : 1358381039000, "i" : 4563 }, "optimeDate" : ISODate("2013-01-17T00:03:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:03Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3208, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:03Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3220, "optime" : { "t" : 1358381043000, "i" : 4948 }, "optimeDate" : ISODate("2013-01-17T00:04:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:04:03.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:04:04.605 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:04:05.432 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 19:04:05.938 [conn192] end connection 127.0.0.2:52848 (3 connections now open) m31002| Wed Jan 16 19:04:06.438 [initandlisten] connection accepted from 127.0.0.2:44696 #195 (4 connections now open) m31000| Wed Jan 16 19:04:06.606 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3226, "optime" : { "t" : 1358381047000, "i" : 6890 }, "optimeDate" : ISODate("2013-01-17T00:04:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3215, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3215, "optime" : { "t" : 1358381043000, "i" : 5669 }, "optimeDate" : ISODate("2013-01-17T00:04:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3225, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3212, "optime" : { "t" : 1358381047000, "i" : 1838 }, "optimeDate" : ISODate("2013-01-17T00:04:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3212, "optime" : { "t" : 1358381043000, "i" : 5669 }, "optimeDate" : ISODate("2013-01-17T00:04:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3212, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:07Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3224, "optime" : { "t" : 1358381047000, "i" : 6890 }, "optimeDate" : ISODate("2013-01-17T00:04:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:04:08.606 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:04:10.047 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:04:10.433 [conn191] end connection 127.0.0.4:53921 (2 connections now open) m31000| Wed Jan 16 19:04:10.606 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:04:10.933 [initandlisten] connection accepted from 127.0.0.4:58900 #193 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3230, "optime" : { "t" : 1358381051000, "i" : 7834 }, "optimeDate" : ISODate("2013-01-17T00:04:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3219, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3219, "optime" : { "t" : 1358381051000, "i" : 3672 }, "optimeDate" : ISODate("2013-01-17T00:04:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:03:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3229, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3216, "optime" : { "t" : 1358381051000, "i" : 1495 }, "optimeDate" : ISODate("2013-01-17T00:04:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3216, "optime" : { "t" : 1358381047000, "i" : 6890 }, "optimeDate" : ISODate("2013-01-17T00:04:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:11Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3216, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3228, "optime" : { "t" : 1358381051000, "i" : 7834 }, "optimeDate" : ISODate("2013-01-17T00:04:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:04:12.607 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:04:14.607 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:04:15.934 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3235, "optime" : { "t" : 1358381056000, "i" : 503 }, "optimeDate" : ISODate("2013-01-17T00:04:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3224, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3224, "optime" : { "t" : 1358381051000, "i" : 7834 }, "optimeDate" : ISODate("2013-01-17T00:04:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3234, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3221, "optime" : { "t" : 1358381055000, "i" : 1800 }, "optimeDate" : ISODate("2013-01-17T00:04:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:15Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3221, "optime" : { "t" : 1358381056000, "i" : 362 }, "optimeDate" : ISODate("2013-01-17T00:04:16Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3221, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3233, "optime" : { "t" : 1358381056000, "i" : 503 }, "optimeDate" : ISODate("2013-01-17T00:04:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:04:16.608 [conn192] end connection 127.0.0.2:38023 (2 connections now open) m31001| Wed Jan 16 19:04:16.608 [initandlisten] connection accepted from 127.0.0.2:51568 #194 (3 connections now open) m31000| Wed Jan 16 19:04:16.608 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:04:18.608 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:04:20.049 [conn117] end connection 127.0.0.4:48025 (4 connections now open) m31000| Wed Jan 16 19:04:20.049 [initandlisten] connection accepted from 127.0.0.4:55160 #118 (5 connections now open) m31002| Wed Jan 16 19:04:20.049 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3239, "optime" : { "t" : 1358381060000, "i" : 1835 }, "optimeDate" : ISODate("2013-01-17T00:04:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3228, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3228, "optime" : { "t" : 1358381056000, "i" : 503 }, "optimeDate" : ISODate("2013-01-17T00:04:16Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3238, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3225, "optime" : { "t" : 1358381059000, "i" : 399 }, "optimeDate" : ISODate("2013-01-17T00:04:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3225, "optime" : { "t" : 1358381060000, "i" : 410 }, "optimeDate" : ISODate("2013-01-17T00:04:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3225, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:19Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3237, "optime" : { "t" : 1358381060000, "i" : 1807 }, "optimeDate" : ISODate("2013-01-17T00:04:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:04:20.609 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:04:22.609 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3243, "optime" : { "t" : 1358381064000, "i" : 3869 }, "optimeDate" : ISODate("2013-01-17T00:04:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3232, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3232, "optime" : { "t" : 1358381063000, "i" : 5880 }, "optimeDate" : ISODate("2013-01-17T00:04:23Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:24Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3242, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3229, "optime" : { "t" : 1358381060000, "i" : 1835 }, "optimeDate" : ISODate("2013-01-17T00:04:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:23Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3229, "optime" : { "t" : 1358381064000, "i" : 973 }, "optimeDate" : ISODate("2013-01-17T00:04:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3229, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3241, "optime" : { "t" : 1358381064000, "i" : 3869 }, "optimeDate" : ISODate("2013-01-17T00:04:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:04:24.610 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:04:25.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:04:25.935 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:04:26.610 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3247, "optime" : { "t" : 1358381068000, "i" : 4736 }, "optimeDate" : ISODate("2013-01-17T00:04:28Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3236, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3236, "optime" : { "t" : 1358381064000, "i" : 3869 }, "optimeDate" : ISODate("2013-01-17T00:04:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:28Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3246, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3233, "optime" : { "t" : 1358381064000, "i" : 3869 }, "optimeDate" : ISODate("2013-01-17T00:04:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:28Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:28Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3233, "optime" : { "t" : 1358381068000, "i" : 637 }, "optimeDate" : ISODate("2013-01-17T00:04:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3233, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3245, "optime" : { "t" : 1358381068000, "i" : 4736 }, "optimeDate" : ISODate("2013-01-17T00:04:28Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:04:28.610 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:04:30.051 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:04:30.611 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:04:32.611 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3251, "optime" : { "t" : 1358381072000, "i" : 6475 }, "optimeDate" : ISODate("2013-01-17T00:04:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3240, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3240, "optime" : { "t" : 1358381068000, "i" : 4736 }, "optimeDate" : ISODate("2013-01-17T00:04:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:32Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3250, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3237, "optime" : { "t" : 1358381068000, "i" : 4736 }, "optimeDate" : ISODate("2013-01-17T00:04:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3237, "optime" : { "t" : 1358381072000, "i" : 1174 }, "optimeDate" : ISODate("2013-01-17T00:04:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3237, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:31Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3249, "optime" : { "t" : 1358381072000, "i" : 6475 }, "optimeDate" : ISODate("2013-01-17T00:04:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:04:33.205 [conn194] end connection 127.0.0.3:54653 (3 connections now open) m31002| Wed Jan 16 19:04:33.205 [initandlisten] connection accepted from 127.0.0.3:33330 #196 (4 connections now open) m31000| Wed Jan 16 19:04:34.612 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:04:35.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:04:35.936 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:04:36.612 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3255, "optime" : { "t" : 1358381076000, "i" : 7216 }, "optimeDate" : ISODate("2013-01-17T00:04:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3244, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3244, "optime" : { "t" : 1358381076000, "i" : 3659 }, "optimeDate" : ISODate("2013-01-17T00:04:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3254, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3241, "optime" : { "t" : 1358381072000, "i" : 6475 }, "optimeDate" : ISODate("2013-01-17T00:04:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:35Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3241, "optime" : { "t" : 1358381076000, "i" : 39 }, "optimeDate" : ISODate("2013-01-17T00:04:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3241, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3253, "optime" : { "t" : 1358381076000, "i" : 7216 }, "optimeDate" : ISODate("2013-01-17T00:04:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:04:38.612 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:04:40.052 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:04:40.613 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3259, "optime" : { "t" : 1358381080000, "i" : 8699 }, "optimeDate" : ISODate("2013-01-17T00:04:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3248, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3248, "optime" : { "t" : 1358381076000, "i" : 7216 }, "optimeDate" : ISODate("2013-01-17T00:04:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:40Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3258, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3245, "optime" : { "t" : 1358381076000, "i" : 7216 }, "optimeDate" : ISODate("2013-01-17T00:04:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:40Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3245, "optime" : { "t" : 1358381080000, "i" : 182 }, "optimeDate" : ISODate("2013-01-17T00:04:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3245, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3257, "optime" : { "t" : 1358381080000, "i" : 8699 }, "optimeDate" : ISODate("2013-01-17T00:04:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:04:42.613 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:04:43.944 [conn195] end connection 127.0.0.2:44696 (3 connections now open) m31002| Wed Jan 16 19:04:44.444 [initandlisten] connection accepted from 127.0.0.2:37998 #197 (4 connections now open) m31000| Wed Jan 16 19:04:44.613 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3264, "optime" : { "t" : 1358381085000, "i" : 1036 }, "optimeDate" : ISODate("2013-01-17T00:04:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3253, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3253, "optime" : { "t" : 1358381084000, "i" : 3940 }, "optimeDate" : ISODate("2013-01-17T00:04:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3263, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3250, "optime" : { "t" : 1358381080000, "i" : 8699 }, "optimeDate" : ISODate("2013-01-17T00:04:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:43Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3250, "optime" : { "t" : 1358381084000, "i" : 374 }, "optimeDate" : ISODate("2013-01-17T00:04:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3250, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3262, "optime" : { "t" : 1358381085000, "i" : 501 }, "optimeDate" : ISODate("2013-01-17T00:04:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:04:45.938 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:04:46.614 [conn194] end connection 127.0.0.2:51568 (2 connections now open) m31001| Wed Jan 16 19:04:46.614 [initandlisten] connection accepted from 127.0.0.2:46061 #195 (3 connections now open) m31000| Wed Jan 16 19:04:46.614 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:04:48.438 [conn193] end connection 127.0.0.4:58900 (2 connections now open) m31000| Wed Jan 16 19:04:48.614 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:04:48.703 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:212807 106ms m31001| Wed Jan 16 19:04:48.938 [initandlisten] connection accepted from 127.0.0.4:40997 #196 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3268, "optime" : { "t" : 1358381089000, "i" : 1888 }, "optimeDate" : ISODate("2013-01-17T00:04:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3257, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3257, "optime" : { "t" : 1358381085000, "i" : 1036 }, "optimeDate" : ISODate("2013-01-17T00:04:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:48Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3267, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3254, "optime" : { "t" : 1358381089000, "i" : 1291 }, "optimeDate" : ISODate("2013-01-17T00:04:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3254, "optime" : { "t" : 1358381085000, "i" : 1036 }, "optimeDate" : ISODate("2013-01-17T00:04:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3254, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3266, "optime" : { "t" : 1358381089000, "i" : 1292 }, "optimeDate" : ISODate("2013-01-17T00:04:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:04:50.054 [conn118] end connection 127.0.0.4:55160 (4 connections now open) m31000| Wed Jan 16 19:04:50.054 [initandlisten] connection accepted from 127.0.0.4:46303 #119 (5 connections now open) m31002| Wed Jan 16 19:04:50.054 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:04:50.615 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:04:52.615 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3272, "optime" : { "t" : 1358381093000, "i" : 2719 }, "optimeDate" : ISODate("2013-01-17T00:04:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3261, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3261, "optime" : { "t" : 1358381089000, "i" : 1888 }, "optimeDate" : ISODate("2013-01-17T00:04:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:52Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3271, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3258, "optime" : { "t" : 1358381093000, "i" : 1841 }, "optimeDate" : ISODate("2013-01-17T00:04:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3258, "optime" : { "t" : 1358381089000, "i" : 1888 }, "optimeDate" : ISODate("2013-01-17T00:04:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3258, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:53Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3270, "optime" : { "t" : 1358381093000, "i" : 2719 }, "optimeDate" : ISODate("2013-01-17T00:04:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:04:54.615 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:04:56.055 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:04:56.616 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3276, "optime" : { "t" : 1358381097000, "i" : 4444 }, "optimeDate" : ISODate("2013-01-17T00:04:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3265, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3265, "optime" : { "t" : 1358381096000, "i" : 4684 }, "optimeDate" : ISODate("2013-01-17T00:04:56Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3275, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3262, "optime" : { "t" : 1358381097000, "i" : 1936 }, "optimeDate" : ISODate("2013-01-17T00:04:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:04:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3262, "optime" : { "t" : 1358381093000, "i" : 2719 }, "optimeDate" : ISODate("2013-01-17T00:04:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3262, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:04:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3274, "optime" : { "t" : 1358381097000, "i" : 4444 }, "optimeDate" : ISODate("2013-01-17T00:04:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:04:57.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:04:58.616 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:05:00.616 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:05:01.440 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3280, "optime" : { "t" : 1358381101000, "i" : 5789 }, "optimeDate" : ISODate("2013-01-17T00:05:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3269, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3269, "optime" : { "t" : 1358381097000, "i" : 4444 }, "optimeDate" : ISODate("2013-01-17T00:04:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3279, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3266, "optime" : { "t" : 1358381101000, "i" : 2084 }, "optimeDate" : ISODate("2013-01-17T00:05:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:01Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3266, "optime" : { "t" : 1358381097000, "i" : 4444 }, "optimeDate" : ISODate("2013-01-17T00:04:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3266, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3278, "optime" : { "t" : 1358381101000, "i" : 5789 }, "optimeDate" : ISODate("2013-01-17T00:05:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:05:02.617 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:05:03.210 [conn196] end connection 127.0.0.3:33330 (3 connections now open) m31002| Wed Jan 16 19:05:03.210 [initandlisten] connection accepted from 127.0.0.3:41008 #198 (4 connections now open) m31000| Wed Jan 16 19:05:04.617 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3284, "optime" : { "t" : 1358381105000, "i" : 6774 }, "optimeDate" : ISODate("2013-01-17T00:05:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3273, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3273, "optime" : { "t" : 1358381101000, "i" : 5789 }, "optimeDate" : ISODate("2013-01-17T00:05:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3283, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3270, "optime" : { "t" : 1358381105000, "i" : 1322 }, "optimeDate" : ISODate("2013-01-17T00:05:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3270, "optime" : { "t" : 1358381101000, "i" : 5789 }, "optimeDate" : ISODate("2013-01-17T00:05:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3270, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:05Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3282, "optime" : { "t" : 1358381105000, "i" : 6774 }, "optimeDate" : ISODate("2013-01-17T00:05:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:05:06.057 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:05:06.617 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:05:07.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:05:08.618 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3288, "optime" : { "t" : 1358381109000, "i" : 7590 }, "optimeDate" : ISODate("2013-01-17T00:05:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3277, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3277, "optime" : { "t" : 1358381109000, "i" : 3678 }, "optimeDate" : ISODate("2013-01-17T00:05:09Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3287, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3274, "optime" : { "t" : 1358381109000, "i" : 1536 }, "optimeDate" : ISODate("2013-01-17T00:05:09Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3274, "optime" : { "t" : 1358381105000, "i" : 6774 }, "optimeDate" : ISODate("2013-01-17T00:05:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:09Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3274, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:09Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3286, "optime" : { "t" : 1358381109000, "i" : 7590 }, "optimeDate" : ISODate("2013-01-17T00:05:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:05:10.618 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:05:11.441 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:05:12.619 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:05:14.013 [journal] old journal file will be removed: /data/db/testReplSet-0/journal/j._4 STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 19:05:14.115 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:329754 164ms { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3293, "optime" : { "t" : 1358381114000, "i" : 1484 }, "optimeDate" : ISODate("2013-01-17T00:05:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3282, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3282, "optime" : { "t" : 1358381109000, "i" : 7590 }, "optimeDate" : ISODate("2013-01-17T00:05:09Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:14Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:04:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3292, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3279, "optime" : { "t" : 1358381113000, "i" : 1679 }, "optimeDate" : ISODate("2013-01-17T00:05:13Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:13Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3279, "optime" : { "t" : 1358381113000, "i" : 8369 }, "optimeDate" : ISODate("2013-01-17T00:05:13Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3279, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3291, "optime" : { "t" : 1358381114000, "i" : 1484 }, "optimeDate" : ISODate("2013-01-17T00:05:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:05:14.619 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:05:16.059 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:05:16.619 [conn195] end connection 127.0.0.2:46061 (2 connections now open) m31001| Wed Jan 16 19:05:16.619 [initandlisten] connection accepted from 127.0.0.2:60232 #197 (3 connections now open) m31000| Wed Jan 16 19:05:16.620 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3297, "optime" : { "t" : 1358381118000, "i" : 2662 }, "optimeDate" : ISODate("2013-01-17T00:05:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3286, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3286, "optime" : { "t" : 1358381114000, "i" : 1484 }, "optimeDate" : ISODate("2013-01-17T00:05:14Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:18Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3296, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3283, "optime" : { "t" : 1358381114000, "i" : 1484 }, "optimeDate" : ISODate("2013-01-17T00:05:14Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3283, "optime" : { "t" : 1358381118000, "i" : 159 }, "optimeDate" : ISODate("2013-01-17T00:05:18Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3283, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3295, "optime" : { "t" : 1358381118000, "i" : 2662 }, "optimeDate" : ISODate("2013-01-17T00:05:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:05:18.620 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:05:20.059 [conn119] end connection 127.0.0.4:46303 (4 connections now open) m31000| Wed Jan 16 19:05:20.060 [initandlisten] connection accepted from 127.0.0.4:38334 #120 (5 connections now open) m31000| Wed Jan 16 19:05:20.620 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:05:21.442 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 19:05:21.949 [conn197] end connection 127.0.0.2:37998 (3 connections now open) m31002| Wed Jan 16 19:05:22.449 [initandlisten] connection accepted from 127.0.0.2:32983 #199 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3301, "optime" : { "t" : 1358381122000, "i" : 4239 }, "optimeDate" : ISODate("2013-01-17T00:05:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3290, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3290, "optime" : { "t" : 1358381122000, "i" : 3610 }, "optimeDate" : ISODate("2013-01-17T00:05:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3300, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3287, "optime" : { "t" : 1358381118000, "i" : 2662 }, "optimeDate" : ISODate("2013-01-17T00:05:18Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:21Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3287, "optime" : { "t" : 1358381122000, "i" : 667 }, "optimeDate" : ISODate("2013-01-17T00:05:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3287, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3299, "optime" : { "t" : 1358381122000, "i" : 4239 }, "optimeDate" : ISODate("2013-01-17T00:05:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:05:22.621 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:05:24.621 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:05:26.061 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:05:26.443 [conn196] end connection 127.0.0.4:40997 (2 connections now open) m31000| Wed Jan 16 19:05:26.622 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3305, "optime" : { "t" : 1358381126000, "i" : 5631 }, "optimeDate" : ISODate("2013-01-17T00:05:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3294, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3294, "optime" : { "t" : 1358381122000, "i" : 4239 }, "optimeDate" : ISODate("2013-01-17T00:05:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:26Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3304, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3291, "optime" : { "t" : 1358381122000, "i" : 4239 }, "optimeDate" : ISODate("2013-01-17T00:05:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3291, "optime" : { "t" : 1358381125000, "i" : 4368 }, "optimeDate" : ISODate("2013-01-17T00:05:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3291, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3303, "optime" : { "t" : 1358381126000, "i" : 5631 }, "optimeDate" : ISODate("2013-01-17T00:05:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:05:26.943 [initandlisten] connection accepted from 127.0.0.4:33778 #198 (3 connections now open) m31000| Wed Jan 16 19:05:28.622 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:05:29.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:05:30.622 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3309, "optime" : { "t" : 1358381130000, "i" : 7494 }, "optimeDate" : ISODate("2013-01-17T00:05:30Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3298, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3298, "optime" : { "t" : 1358381129000, "i" : 1740 }, "optimeDate" : ISODate("2013-01-17T00:05:29Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:30Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3308, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3295, "optime" : { "t" : 1358381126000, "i" : 5631 }, "optimeDate" : ISODate("2013-01-17T00:05:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:29Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3295, "optime" : { "t" : 1358381130000, "i" : 475 }, "optimeDate" : ISODate("2013-01-17T00:05:30Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3295, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3307, "optime" : { "t" : 1358381130000, "i" : 7262 }, "optimeDate" : ISODate("2013-01-17T00:05:30Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:05:31.944 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:05:32.623 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:05:33.216 [conn198] end connection 127.0.0.3:41008 (3 connections now open) m31002| Wed Jan 16 19:05:33.216 [initandlisten] connection accepted from 127.0.0.3:42873 #200 (4 connections now open) m31000| Wed Jan 16 19:05:34.623 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3314, "optime" : { "t" : 1358381135000, "i" : 558 }, "optimeDate" : ISODate("2013-01-17T00:05:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3303, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3303, "optime" : { "t" : 1358381134000, "i" : 8130 }, "optimeDate" : ISODate("2013-01-17T00:05:34Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3313, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3300, "optime" : { "t" : 1358381130000, "i" : 7494 }, "optimeDate" : ISODate("2013-01-17T00:05:30Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:34Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3300, "optime" : { "t" : 1358381134000, "i" : 786 }, "optimeDate" : ISODate("2013-01-17T00:05:34Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3300, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3312, "optime" : { "t" : 1358381135000, "i" : 385 }, "optimeDate" : ISODate("2013-01-17T00:05:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:05:36.062 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:05:36.623 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:05:38.624 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3318, "optime" : { "t" : 1358381139000, "i" : 2046 }, "optimeDate" : ISODate("2013-01-17T00:05:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3307, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3307, "optime" : { "t" : 1358381135000, "i" : 558 }, "optimeDate" : ISODate("2013-01-17T00:05:35Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:38Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3317, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3304, "optime" : { "t" : 1358381139000, "i" : 1592 }, "optimeDate" : ISODate("2013-01-17T00:05:39Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3304, "optime" : { "t" : 1358381135000, "i" : 558 }, "optimeDate" : ISODate("2013-01-17T00:05:35Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3304, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:39Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3316, "optime" : { "t" : 1358381139000, "i" : 1592 }, "optimeDate" : ISODate("2013-01-17T00:05:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:05:39.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:05:40.624 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:05:41.945 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:05:42.624 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3322, "optime" : { "t" : 1358381143000, "i" : 2815 }, "optimeDate" : ISODate("2013-01-17T00:05:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3311, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3311, "optime" : { "t" : 1358381142000, "i" : 1793 }, "optimeDate" : ISODate("2013-01-17T00:05:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3321, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3308, "optime" : { "t" : 1358381143000, "i" : 1783 }, "optimeDate" : ISODate("2013-01-17T00:05:43Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3308, "optime" : { "t" : 1358381139000, "i" : 2046 }, "optimeDate" : ISODate("2013-01-17T00:05:39Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3308, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:43Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3320, "optime" : { "t" : 1358381143000, "i" : 2815 }, "optimeDate" : ISODate("2013-01-17T00:05:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:05:44.625 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:05:46.064 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:05:46.625 [conn197] end connection 127.0.0.2:60232 (2 connections now open) m31001| Wed Jan 16 19:05:46.625 [initandlisten] connection accepted from 127.0.0.2:42071 #199 (3 connections now open) m31000| Wed Jan 16 19:05:46.625 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3326, "optime" : { "t" : 1358381147000, "i" : 4034 }, "optimeDate" : ISODate("2013-01-17T00:05:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3315, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3315, "optime" : { "t" : 1358381147000, "i" : 3647 }, "optimeDate" : ISODate("2013-01-17T00:05:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3325, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3312, "optime" : { "t" : 1358381147000, "i" : 1993 }, "optimeDate" : ISODate("2013-01-17T00:05:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3312, "optime" : { "t" : 1358381143000, "i" : 2815 }, "optimeDate" : ISODate("2013-01-17T00:05:43Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:47Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3312, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3324, "optime" : { "t" : 1358381147000, "i" : 3648 }, "optimeDate" : ISODate("2013-01-17T00:05:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:05:48.626 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:05:50.065 [conn120] end connection 127.0.0.4:38334 (4 connections now open) m31000| Wed Jan 16 19:05:50.065 [initandlisten] connection accepted from 127.0.0.4:44569 #121 (5 connections now open) m31000| Wed Jan 16 19:05:50.626 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3330, "optime" : { "t" : 1358381151000, "i" : 6016 }, "optimeDate" : ISODate("2013-01-17T00:05:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3319, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3319, "optime" : { "t" : 1358381147000, "i" : 4034 }, "optimeDate" : ISODate("2013-01-17T00:05:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:50Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3329, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3316, "optime" : { "t" : 1358381151000, "i" : 2250 }, "optimeDate" : ISODate("2013-01-17T00:05:51Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3316, "optime" : { "t" : 1358381147000, "i" : 4034 }, "optimeDate" : ISODate("2013-01-17T00:05:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3316, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3328, "optime" : { "t" : 1358381151000, "i" : 6016 }, "optimeDate" : ISODate("2013-01-17T00:05:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:05:51.947 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:05:52.626 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:05:54.627 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3334, "optime" : { "t" : 1358381155000, "i" : 7338 }, "optimeDate" : ISODate("2013-01-17T00:05:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3323, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3323, "optime" : { "t" : 1358381154000, "i" : 2043 }, "optimeDate" : ISODate("2013-01-17T00:05:54Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3333, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3320, "optime" : { "t" : 1358381155000, "i" : 1591 }, "optimeDate" : ISODate("2013-01-17T00:05:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:05:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3320, "optime" : { "t" : 1358381151000, "i" : 6016 }, "optimeDate" : ISODate("2013-01-17T00:05:51Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3320, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3332, "optime" : { "t" : 1358381155000, "i" : 7338 }, "optimeDate" : ISODate("2013-01-17T00:05:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:05:56.066 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:05:56.627 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:05:58.628 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:05:59.954 [conn199] end connection 127.0.0.2:32983 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3339, "optime" : { "t" : 1358381160000, "i" : 250 }, "optimeDate" : ISODate("2013-01-17T00:06:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3328, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3328, "optime" : { "t" : 1358381155000, "i" : 7338 }, "optimeDate" : ISODate("2013-01-17T00:05:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:58Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3338, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3325, "optime" : { "t" : 1358381159000, "i" : 1651 }, "optimeDate" : ISODate("2013-01-17T00:05:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:05:59Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3325, "optime" : { "t" : 1358381155000, "i" : 7338 }, "optimeDate" : ISODate("2013-01-17T00:05:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3325, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3337, "optime" : { "t" : 1358381160000, "i" : 62 }, "optimeDate" : ISODate("2013-01-17T00:06:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:06:00.455 [initandlisten] connection accepted from 127.0.0.2:52930 #201 (4 connections now open) m31000| Wed Jan 16 19:06:00.628 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:06:01.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:06:01.948 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:06:02.628 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:06:03.221 [conn200] end connection 127.0.0.3:42873 (3 connections now open) m31002| Wed Jan 16 19:06:03.221 [initandlisten] connection accepted from 127.0.0.3:49000 #202 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3343, "optime" : { "t" : 1358381164000, "i" : 1449 }, "optimeDate" : ISODate("2013-01-17T00:06:04Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3332, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3332, "optime" : { "t" : 1358381160000, "i" : 250 }, "optimeDate" : ISODate("2013-01-17T00:06:00Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:04Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3342, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3329, "optime" : { "t" : 1358381163000, "i" : 1548 }, "optimeDate" : ISODate("2013-01-17T00:06:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3329, "optime" : { "t" : 1358381164000, "i" : 474 }, "optimeDate" : ISODate("2013-01-17T00:06:04Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3329, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:03Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3341, "optime" : { "t" : 1358381164000, "i" : 1449 }, "optimeDate" : ISODate("2013-01-17T00:06:04Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:06:04.448 [conn198] end connection 127.0.0.4:33778 (2 connections now open) m31000| Wed Jan 16 19:06:04.629 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:06:04.948 [initandlisten] connection accepted from 127.0.0.4:48541 #200 (3 connections now open) m31002| Wed Jan 16 19:06:06.068 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:06:06.629 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3347, "optime" : { "t" : 1358381168000, "i" : 2624 }, "optimeDate" : ISODate("2013-01-17T00:06:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3336, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3336, "optime" : { "t" : 1358381167000, "i" : 6940 }, "optimeDate" : ISODate("2013-01-17T00:06:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:08Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3346, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3333, "optime" : { "t" : 1358381167000, "i" : 256 }, "optimeDate" : ISODate("2013-01-17T00:06:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:07Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3333, "optime" : { "t" : 1358381168000, "i" : 769 }, "optimeDate" : ISODate("2013-01-17T00:06:08Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3333, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3345, "optime" : { "t" : 1358381168000, "i" : 2624 }, "optimeDate" : ISODate("2013-01-17T00:06:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:06:08.630 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:06:10.630 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:06:11.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:06:12.069 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3351, "optime" : { "t" : 1358381172000, "i" : 3672 }, "optimeDate" : ISODate("2013-01-17T00:06:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3340, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3340, "optime" : { "t" : 1358381168000, "i" : 2624 }, "optimeDate" : ISODate("2013-01-17T00:06:08Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:12Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3350, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3337, "optime" : { "t" : 1358381168000, "i" : 2624 }, "optimeDate" : ISODate("2013-01-17T00:06:08Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3337, "optime" : { "t" : 1358381172000, "i" : 896 }, "optimeDate" : ISODate("2013-01-17T00:06:12Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3337, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3349, "optime" : { "t" : 1358381172000, "i" : 3672 }, "optimeDate" : ISODate("2013-01-17T00:06:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:06:12.630 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:06:14.631 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 m31001| Wed Jan 16 19:06:16.631 [conn199] end connection 127.0.0.2:42071 (2 connections now open) m31001| Wed Jan 16 19:06:16.631 [initandlisten] connection accepted from 127.0.0.2:58711 #201 (3 connections now open) m31000| Wed Jan 16 19:06:16.631 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3355, "optime" : { "t" : 1358381176000, "i" : 5308 }, "optimeDate" : ISODate("2013-01-17T00:06:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3344, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3344, "optime" : { "t" : 1358381175000, "i" : 456 }, "optimeDate" : ISODate("2013-01-17T00:06:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3354, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3341, "optime" : { "t" : 1358381172000, "i" : 3672 }, "optimeDate" : ISODate("2013-01-17T00:06:12Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3341, "optime" : { "t" : 1358381176000, "i" : 1123 }, "optimeDate" : ISODate("2013-01-17T00:06:16Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3341, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:15Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3353, "optime" : { "t" : 1358381176000, "i" : 5308 }, "optimeDate" : ISODate("2013-01-17T00:06:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:06:17.450 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:06:18.632 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:06:20.070 [conn121] end connection 127.0.0.4:44569 (4 connections now open) m31000| Wed Jan 16 19:06:20.070 [initandlisten] connection accepted from 127.0.0.4:40116 #122 (5 connections now open) m31000| Wed Jan 16 19:06:20.632 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3359, "optime" : { "t" : 1358381180000, "i" : 7019 }, "optimeDate" : ISODate("2013-01-17T00:06:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3348, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3348, "optime" : { "t" : 1358381180000, "i" : 3627 }, "optimeDate" : ISODate("2013-01-17T00:06:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:05:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3358, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3345, "optime" : { "t" : 1358381176000, "i" : 5308 }, "optimeDate" : ISODate("2013-01-17T00:06:16Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:19Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3345, "optime" : { "t" : 1358381180000, "i" : 927 }, "optimeDate" : ISODate("2013-01-17T00:06:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3345, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3357, "optime" : { "t" : 1358381180000, "i" : 6619 }, "optimeDate" : ISODate("2013-01-17T00:06:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:06:20.825 [journal] old journal file will be removed: /data/db/testReplSet-2/journal/j._4 m31002| Wed Jan 16 19:06:22.071 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:06:22.632 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:06:24.633 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3363, "optime" : { "t" : 1358381184000, "i" : 8405 }, "optimeDate" : ISODate("2013-01-17T00:06:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3352, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3352, "optime" : { "t" : 1358381180000, "i" : 7019 }, "optimeDate" : ISODate("2013-01-17T00:06:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:24Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3362, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3349, "optime" : { "t" : 1358381180000, "i" : 7019 }, "optimeDate" : ISODate("2013-01-17T00:06:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3349, "optime" : { "t" : 1358381184000, "i" : 904 }, "optimeDate" : ISODate("2013-01-17T00:06:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3349, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3361, "optime" : { "t" : 1358381184000, "i" : 8405 }, "optimeDate" : ISODate("2013-01-17T00:06:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:06:26.633 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:06:27.452 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:06:28.633 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:06:28.958 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:945778 472ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3368, "optime" : { "t" : 1358381189000, "i" : 4018 }, "optimeDate" : ISODate("2013-01-17T00:06:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3357, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3357, "optime" : { "t" : 1358381187000, "i" : 261 }, "optimeDate" : ISODate("2013-01-17T00:06:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:28Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3367, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3354, "optime" : { "t" : 1358381184000, "i" : 8405 }, "optimeDate" : ISODate("2013-01-17T00:06:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:27Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3354, "optime" : { "t" : 1358381188000, "i" : 278 }, "optimeDate" : ISODate("2013-01-17T00:06:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3354, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3366, "optime" : { "t" : 1358381189000, "i" : 3945 }, "optimeDate" : ISODate("2013-01-17T00:06:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:06:30.634 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:06:32.072 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:06:32.634 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:06:33.227 [conn202] end connection 127.0.0.3:49000 (3 connections now open) m31002| Wed Jan 16 19:06:33.227 [initandlisten] connection accepted from 127.0.0.3:38517 #203 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3372, "optime" : { "t" : 1358381193000, "i" : 3238 }, "optimeDate" : ISODate("2013-01-17T00:06:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3361, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3361, "optime" : { "t" : 1358381192000, "i" : 6150 }, "optimeDate" : ISODate("2013-01-17T00:06:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3371, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3358, "optime" : { "t" : 1358381193000, "i" : 1898 }, "optimeDate" : ISODate("2013-01-17T00:06:33Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3358, "optime" : { "t" : 1358381189000, "i" : 4018 }, "optimeDate" : ISODate("2013-01-17T00:06:29Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3358, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:33Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3370, "optime" : { "t" : 1358381193000, "i" : 3238 }, "optimeDate" : ISODate("2013-01-17T00:06:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:06:33.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:06:34.634 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:06:36.635 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:06:37.453 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3376, "optime" : { "t" : 1358381197000, "i" : 4958 }, "optimeDate" : ISODate("2013-01-17T00:06:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3365, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3365, "optime" : { "t" : 1358381193000, "i" : 3238 }, "optimeDate" : ISODate("2013-01-17T00:06:33Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:36Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3375, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3362, "optime" : { "t" : 1358381197000, "i" : 1680 }, "optimeDate" : ISODate("2013-01-17T00:06:37Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:37Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3362, "optime" : { "t" : 1358381193000, "i" : 3238 }, "optimeDate" : ISODate("2013-01-17T00:06:33Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3362, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3374, "optime" : { "t" : 1358381197000, "i" : 4958 }, "optimeDate" : ISODate("2013-01-17T00:06:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:06:37.960 [conn201] end connection 127.0.0.2:52930 (3 connections now open) m31002| Wed Jan 16 19:06:38.460 [initandlisten] connection accepted from 127.0.0.2:42164 #204 (4 connections now open) m31000| Wed Jan 16 19:06:38.635 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:06:40.635 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3380, "optime" : { "t" : 1358381201000, "i" : 6930 }, "optimeDate" : ISODate("2013-01-17T00:06:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3369, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3369, "optime" : { "t" : 1358381200000, "i" : 2778 }, "optimeDate" : ISODate("2013-01-17T00:06:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3379, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3366, "optime" : { "t" : 1358381201000, "i" : 1973 }, "optimeDate" : ISODate("2013-01-17T00:06:41Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:41Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3366, "optime" : { "t" : 1358381197000, "i" : 4958 }, "optimeDate" : ISODate("2013-01-17T00:06:37Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3366, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:41Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3378, "optime" : { "t" : 1358381201000, "i" : 6318 }, "optimeDate" : ISODate("2013-01-17T00:06:41Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:06:42.074 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:06:42.453 [conn200] end connection 127.0.0.4:48541 (2 connections now open) m31000| Wed Jan 16 19:06:42.636 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:06:42.954 [initandlisten] connection accepted from 127.0.0.4:57742 #202 (3 connections now open) m31001| Wed Jan 16 19:06:43.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:06:44.636 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3384, "optime" : { "t" : 1358381205000, "i" : 8254 }, "optimeDate" : ISODate("2013-01-17T00:06:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3373, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3373, "optime" : { "t" : 1358381205000, "i" : 8046 }, "optimeDate" : ISODate("2013-01-17T00:06:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3383, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3370, "optime" : { "t" : 1358381205000, "i" : 1805 }, "optimeDate" : ISODate("2013-01-17T00:06:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:45Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3370, "optime" : { "t" : 1358381201000, "i" : 6930 }, "optimeDate" : ISODate("2013-01-17T00:06:41Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:45Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3370, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3382, "optime" : { "t" : 1358381205000, "i" : 8046 }, "optimeDate" : ISODate("2013-01-17T00:06:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:06:46.637 [conn201] end connection 127.0.0.2:58711 (2 connections now open) m31001| Wed Jan 16 19:06:46.637 [initandlisten] connection accepted from 127.0.0.2:41817 #203 (3 connections now open) m31000| Wed Jan 16 19:06:46.637 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:06:47.955 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:06:48.637 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:06:50.075 [conn122] end connection 127.0.0.4:40116 (4 connections now open) m31000| Wed Jan 16 19:06:50.075 [initandlisten] connection accepted from 127.0.0.4:33856 #123 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3389, "optime" : { "t" : 1358381210000, "i" : 807 }, "optimeDate" : ISODate("2013-01-17T00:06:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3378, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3378, "optime" : { "t" : 1358381205000, "i" : 8254 }, "optimeDate" : ISODate("2013-01-17T00:06:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:50Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3388, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3375, "optime" : { "t" : 1358381209000, "i" : 1547 }, "optimeDate" : ISODate("2013-01-17T00:06:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3375, "optime" : { "t" : 1358381210000, "i" : 266 }, "optimeDate" : ISODate("2013-01-17T00:06:50Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3375, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3387, "optime" : { "t" : 1358381210000, "i" : 217 }, "optimeDate" : ISODate("2013-01-17T00:06:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:06:50.638 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:06:52.076 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:06:52.638 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3393, "optime" : { "t" : 1358381214000, "i" : 2621 }, "optimeDate" : ISODate("2013-01-17T00:06:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3382, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3382, "optime" : { "t" : 1358381213000, "i" : 3212 }, "optimeDate" : ISODate("2013-01-17T00:06:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:54Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3392, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3379, "optime" : { "t" : 1358381213000, "i" : 470 }, "optimeDate" : ISODate("2013-01-17T00:06:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3379, "optime" : { "t" : 1358381214000, "i" : 1012 }, "optimeDate" : ISODate("2013-01-17T00:06:54Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3379, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:53Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3391, "optime" : { "t" : 1358381214000, "i" : 2621 }, "optimeDate" : ISODate("2013-01-17T00:06:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:06:54.639 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:06:56.639 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:06:57.956 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3397, "optime" : { "t" : 1358381218000, "i" : 4035 }, "optimeDate" : ISODate("2013-01-17T00:06:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3386, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3386, "optime" : { "t" : 1358381218000, "i" : 3837 }, "optimeDate" : ISODate("2013-01-17T00:06:58Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3396, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3383, "optime" : { "t" : 1358381214000, "i" : 2621 }, "optimeDate" : ISODate("2013-01-17T00:06:54Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:57Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:06:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3383, "optime" : { "t" : 1358381218000, "i" : 286 }, "optimeDate" : ISODate("2013-01-17T00:06:58Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:06:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3383, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3395, "optime" : { "t" : 1358381218000, "i" : 4035 }, "optimeDate" : ISODate("2013-01-17T00:06:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:06:58.640 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:07:00.640 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:07:02.077 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3401, "optime" : { "t" : 1358381222000, "i" : 5595 }, "optimeDate" : ISODate("2013-01-17T00:07:02Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3390, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3390, "optime" : { "t" : 1358381218000, "i" : 4035 }, "optimeDate" : ISODate("2013-01-17T00:06:58Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:02Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3400, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3387, "optime" : { "t" : 1358381218000, "i" : 4035 }, "optimeDate" : ISODate("2013-01-17T00:06:58Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3387, "optime" : { "t" : 1358381222000, "i" : 343 }, "optimeDate" : ISODate("2013-01-17T00:07:02Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3387, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3399, "optime" : { "t" : 1358381222000, "i" : 5216 }, "optimeDate" : ISODate("2013-01-17T00:07:02Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:07:02.640 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:07:03.232 [conn203] end connection 127.0.0.3:38517 (3 connections now open) m31002| Wed Jan 16 19:07:03.232 [initandlisten] connection accepted from 127.0.0.3:45966 #205 (4 connections now open) m31000| Wed Jan 16 19:07:04.641 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:07:05.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:07:06.641 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3405, "optime" : { "t" : 1358381226000, "i" : 6828 }, "optimeDate" : ISODate("2013-01-17T00:07:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3394, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3394, "optime" : { "t" : 1358381225000, "i" : 3020 }, "optimeDate" : ISODate("2013-01-17T00:07:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:06Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3404, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3391, "optime" : { "t" : 1358381222000, "i" : 5595 }, "optimeDate" : ISODate("2013-01-17T00:07:02Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:05Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3391, "optime" : { "t" : 1358381226000, "i" : 820 }, "optimeDate" : ISODate("2013-01-17T00:07:06Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3391, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3403, "optime" : { "t" : 1358381226000, "i" : 6828 }, "optimeDate" : ISODate("2013-01-17T00:07:06Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:07:07.957 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:07:08.641 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:07:10.642 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3409, "optime" : { "t" : 1358381230000, "i" : 8833 }, "optimeDate" : ISODate("2013-01-17T00:07:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3398, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3398, "optime" : { "t" : 1358381230000, "i" : 8538 }, "optimeDate" : ISODate("2013-01-17T00:07:10Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3408, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3395, "optime" : { "t" : 1358381226000, "i" : 6828 }, "optimeDate" : ISODate("2013-01-17T00:07:06Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:10Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3395, "optime" : { "t" : 1358381230000, "i" : 396 }, "optimeDate" : ISODate("2013-01-17T00:07:10Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3395, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3407, "optime" : { "t" : 1358381230000, "i" : 8781 }, "optimeDate" : ISODate("2013-01-17T00:07:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:07:12.079 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:07:12.642 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:07:14.642 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3414, "optime" : { "t" : 1358381235000, "i" : 1135 }, "optimeDate" : ISODate("2013-01-17T00:07:15Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3403, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3403, "optime" : { "t" : 1358381230000, "i" : 8833 }, "optimeDate" : ISODate("2013-01-17T00:07:10Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:14Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3413, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3400, "optime" : { "t" : 1358381230000, "i" : 8833 }, "optimeDate" : ISODate("2013-01-17T00:07:10Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3400, "optime" : { "t" : 1358381234000, "i" : 630 }, "optimeDate" : ISODate("2013-01-17T00:07:14Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3400, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:13Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3412, "optime" : { "t" : 1358381235000, "i" : 1135 }, "optimeDate" : ISODate("2013-01-17T00:07:15Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:07:15.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:07:15.965 [conn204] end connection 127.0.0.2:42164 (3 connections now open) m31002| Wed Jan 16 19:07:16.465 [initandlisten] connection accepted from 127.0.0.2:57160 #206 (4 connections now open) m31001| Wed Jan 16 19:07:16.643 [conn203] end connection 127.0.0.2:41817 (2 connections now open) m31001| Wed Jan 16 19:07:16.643 [initandlisten] connection accepted from 127.0.0.2:52920 #204 (3 connections now open) m31000| Wed Jan 16 19:07:16.643 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:07:17.959 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:07:18.643 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3418, "optime" : { "t" : 1358381239000, "i" : 2374 }, "optimeDate" : ISODate("2013-01-17T00:07:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3407, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3407, "optime" : { "t" : 1358381238000, "i" : 7338 }, "optimeDate" : ISODate("2013-01-17T00:07:18Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3417, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3404, "optime" : { "t" : 1358381239000, "i" : 1529 }, "optimeDate" : ISODate("2013-01-17T00:07:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3404, "optime" : { "t" : 1358381235000, "i" : 1135 }, "optimeDate" : ISODate("2013-01-17T00:07:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3404, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:19Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3416, "optime" : { "t" : 1358381239000, "i" : 1537 }, "optimeDate" : ISODate("2013-01-17T00:07:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:07:20.080 [conn123] end connection 127.0.0.4:33856 (4 connections now open) m31000| Wed Jan 16 19:07:20.080 [initandlisten] connection accepted from 127.0.0.4:47579 #124 (5 connections now open) m31001| Wed Jan 16 19:07:20.459 [conn202] end connection 127.0.0.4:57742 (2 connections now open) m31000| Wed Jan 16 19:07:20.644 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:07:20.959 [initandlisten] connection accepted from 127.0.0.4:39408 #205 (3 connections now open) m31002| Wed Jan 16 19:07:22.081 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:07:22.644 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3422, "optime" : { "t" : 1358381243000, "i" : 3964 }, "optimeDate" : ISODate("2013-01-17T00:07:23Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3411, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3411, "optime" : { "t" : 1358381239000, "i" : 2374 }, "optimeDate" : ISODate("2013-01-17T00:07:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:22Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:06:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3421, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3408, "optime" : { "t" : 1358381243000, "i" : 2610 }, "optimeDate" : ISODate("2013-01-17T00:07:23Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3408, "optime" : { "t" : 1358381239000, "i" : 2374 }, "optimeDate" : ISODate("2013-01-17T00:07:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3408, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3420, "optime" : { "t" : 1358381243000, "i" : 3964 }, "optimeDate" : ISODate("2013-01-17T00:07:23Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:07:24.644 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:07:26.645 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3426, "optime" : { "t" : 1358381247000, "i" : 4880 }, "optimeDate" : ISODate("2013-01-17T00:07:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3415, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3415, "optime" : { "t" : 1358381246000, "i" : 223 }, "optimeDate" : ISODate("2013-01-17T00:07:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3425, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3412, "optime" : { "t" : 1358381247000, "i" : 1840 }, "optimeDate" : ISODate("2013-01-17T00:07:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3412, "optime" : { "t" : 1358381243000, "i" : 3964 }, "optimeDate" : ISODate("2013-01-17T00:07:23Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3412, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:27Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3424, "optime" : { "t" : 1358381247000, "i" : 4765 }, "optimeDate" : ISODate("2013-01-17T00:07:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:07:28.082 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:07:28.645 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:07:30.645 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3430, "optime" : { "t" : 1358381251000, "i" : 6225 }, "optimeDate" : ISODate("2013-01-17T00:07:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3419, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3419, "optime" : { "t" : 1358381251000, "i" : 4423 }, "optimeDate" : ISODate("2013-01-17T00:07:31Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3429, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3416, "optime" : { "t" : 1358381251000, "i" : 1864 }, "optimeDate" : ISODate("2013-01-17T00:07:31Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3416, "optime" : { "t" : 1358381247000, "i" : 4880 }, "optimeDate" : ISODate("2013-01-17T00:07:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:31Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3416, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:31Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3428, "optime" : { "t" : 1358381251000, "i" : 5941 }, "optimeDate" : ISODate("2013-01-17T00:07:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:07:32.646 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:07:33.237 [conn205] end connection 127.0.0.3:45966 (3 connections now open) m31002| Wed Jan 16 19:07:33.237 [initandlisten] connection accepted from 127.0.0.3:57165 #207 (4 connections now open) m31002| Wed Jan 16 19:07:33.461 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:07:34.646 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3434, "optime" : { "t" : 1358381255000, "i" : 7027 }, "optimeDate" : ISODate("2013-01-17T00:07:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3423, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3423, "optime" : { "t" : 1358381251000, "i" : 6225 }, "optimeDate" : ISODate("2013-01-17T00:07:31Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:34Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3433, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3420, "optime" : { "t" : 1358381255000, "i" : 2121 }, "optimeDate" : ISODate("2013-01-17T00:07:35Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3420, "optime" : { "t" : 1358381251000, "i" : 6225 }, "optimeDate" : ISODate("2013-01-17T00:07:31Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3420, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:35Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3432, "optime" : { "t" : 1358381255000, "i" : 7027 }, "optimeDate" : ISODate("2013-01-17T00:07:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:07:36.647 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:07:37.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:07:38.084 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:07:38.647 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:07:38.950 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:250273 125ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3438, "optime" : { "t" : 1358381259000, "i" : 8751 }, "optimeDate" : ISODate("2013-01-17T00:07:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3427, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3427, "optime" : { "t" : 1358381258000, "i" : 136 }, "optimeDate" : ISODate("2013-01-17T00:07:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3437, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3424, "optime" : { "t" : 1358381259000, "i" : 2364 }, "optimeDate" : ISODate("2013-01-17T00:07:39Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3424, "optime" : { "t" : 1358381255000, "i" : 7027 }, "optimeDate" : ISODate("2013-01-17T00:07:35Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3424, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:39Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3436, "optime" : { "t" : 1358381259000, "i" : 8434 }, "optimeDate" : ISODate("2013-01-17T00:07:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:07:40.647 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:07:42.648 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:07:43.462 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:07:43.656 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:353251 176ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3443, "optime" : { "t" : 1358381264000, "i" : 634 }, "optimeDate" : ISODate("2013-01-17T00:07:44Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3432, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3432, "optime" : { "t" : 1358381263000, "i" : 8759 }, "optimeDate" : ISODate("2013-01-17T00:07:43Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3442, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3429, "optime" : { "t" : 1358381263000, "i" : 1961 }, "optimeDate" : ISODate("2013-01-17T00:07:43Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:43Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3429, "optime" : { "t" : 1358381259000, "i" : 8751 }, "optimeDate" : ISODate("2013-01-17T00:07:39Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:43Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3429, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3441, "optime" : { "t" : 1358381264000, "i" : 634 }, "optimeDate" : ISODate("2013-01-17T00:07:44Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:07:44.648 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:07:46.649 [conn204] end connection 127.0.0.2:52920 (2 connections now open) m31001| Wed Jan 16 19:07:46.649 [initandlisten] connection accepted from 127.0.0.2:45212 #206 (3 connections now open) m31000| Wed Jan 16 19:07:46.649 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:07:47.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:07:48.086 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3447, "optime" : { "t" : 1358381268000, "i" : 1899 }, "optimeDate" : ISODate("2013-01-17T00:07:48Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3436, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3436, "optime" : { "t" : 1358381264000, "i" : 634 }, "optimeDate" : ISODate("2013-01-17T00:07:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:48Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3446, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3433, "optime" : { "t" : 1358381267000, "i" : 1276 }, "optimeDate" : ISODate("2013-01-17T00:07:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3433, "optime" : { "t" : 1358381268000, "i" : 396 }, "optimeDate" : ISODate("2013-01-17T00:07:48Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3433, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3445, "optime" : { "t" : 1358381268000, "i" : 1899 }, "optimeDate" : ISODate("2013-01-17T00:07:48Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:07:48.649 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:07:50.086 [conn124] end connection 127.0.0.4:47579 (4 connections now open) m31000| Wed Jan 16 19:07:50.086 [initandlisten] connection accepted from 127.0.0.4:50634 #125 (5 connections now open) m31000| Wed Jan 16 19:07:50.650 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3451, "optime" : { "t" : 1358381272000, "i" : 3628 }, "optimeDate" : ISODate("2013-01-17T00:07:52Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3440, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3440, "optime" : { "t" : 1358381271000, "i" : 1550 }, "optimeDate" : ISODate("2013-01-17T00:07:51Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:52Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3450, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3437, "optime" : { "t" : 1358381268000, "i" : 1899 }, "optimeDate" : ISODate("2013-01-17T00:07:48Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3437, "optime" : { "t" : 1358381272000, "i" : 1184 }, "optimeDate" : ISODate("2013-01-17T00:07:52Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3437, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3449, "optime" : { "t" : 1358381272000, "i" : 3628 }, "optimeDate" : ISODate("2013-01-17T00:07:52Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:07:52.650 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:07:53.464 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 19:07:53.970 [conn206] end connection 127.0.0.2:57160 (3 connections now open) m31002| Wed Jan 16 19:07:54.471 [initandlisten] connection accepted from 127.0.0.2:35010 #208 (4 connections now open) m31000| Wed Jan 16 19:07:54.651 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3455, "optime" : { "t" : 1358381276000, "i" : 4562 }, "optimeDate" : ISODate("2013-01-17T00:07:56Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3444, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3444, "optime" : { "t" : 1358381272000, "i" : 3628 }, "optimeDate" : ISODate("2013-01-17T00:07:52Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:56Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3454, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3441, "optime" : { "t" : 1358381272000, "i" : 3628 }, "optimeDate" : ISODate("2013-01-17T00:07:52Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:55Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:07:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3441, "optime" : { "t" : 1358381276000, "i" : 1137 }, "optimeDate" : ISODate("2013-01-17T00:07:56Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3441, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3453, "optime" : { "t" : 1358381276000, "i" : 3899 }, "optimeDate" : ISODate("2013-01-17T00:07:56Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:07:56.651 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:07:58.087 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:07:58.464 [conn205] end connection 127.0.0.4:39408 (2 connections now open) m31000| Wed Jan 16 19:07:58.651 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:07:58.964 [initandlisten] connection accepted from 127.0.0.4:47947 #207 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3459, "optime" : { "t" : 1358381280000, "i" : 5352 }, "optimeDate" : ISODate("2013-01-17T00:08:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3448, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3448, "optime" : { "t" : 1358381276000, "i" : 4562 }, "optimeDate" : ISODate("2013-01-17T00:07:56Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3458, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3445, "optime" : { "t" : 1358381276000, "i" : 4562 }, "optimeDate" : ISODate("2013-01-17T00:07:56Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3445, "optime" : { "t" : 1358381280000, "i" : 1275 }, "optimeDate" : ISODate("2013-01-17T00:08:00Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3445, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:07:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3457, "optime" : { "t" : 1358381280000, "i" : 5308 }, "optimeDate" : ISODate("2013-01-17T00:08:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:08:00.652 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:08:02.652 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:08:03.242 [conn207] end connection 127.0.0.3:57165 (3 connections now open) m31002| Wed Jan 16 19:08:03.243 [initandlisten] connection accepted from 127.0.0.3:39576 #209 (4 connections now open) m31000| Wed Jan 16 19:08:03.667 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:206606 103ms m31002| Wed Jan 16 19:08:03.965 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:08:04.652 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3463, "optime" : { "t" : 1358381284000, "i" : 6340 }, "optimeDate" : ISODate("2013-01-17T00:08:04Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3452, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3452, "optime" : { "t" : 1358381284000, "i" : 4061 }, "optimeDate" : ISODate("2013-01-17T00:08:04Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3462, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3449, "optime" : { "t" : 1358381280000, "i" : 5352 }, "optimeDate" : ISODate("2013-01-17T00:08:00Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:03Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3449, "optime" : { "t" : 1358381284000, "i" : 1340 }, "optimeDate" : ISODate("2013-01-17T00:08:04Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3449, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3461, "optime" : { "t" : 1358381284000, "i" : 6340 }, "optimeDate" : ISODate("2013-01-17T00:08:04Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:08:06.653 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:08:08.089 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:08:08.653 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3467, "optime" : { "t" : 1358381288000, "i" : 7426 }, "optimeDate" : ISODate("2013-01-17T00:08:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3456, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3456, "optime" : { "t" : 1358381284000, "i" : 6340 }, "optimeDate" : ISODate("2013-01-17T00:08:04Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:08Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3466, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3453, "optime" : { "t" : 1358381284000, "i" : 6340 }, "optimeDate" : ISODate("2013-01-17T00:08:04Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3453, "optime" : { "t" : 1358381288000, "i" : 995 }, "optimeDate" : ISODate("2013-01-17T00:08:08Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3453, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:07Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3465, "optime" : { "t" : 1358381288000, "i" : 6974 }, "optimeDate" : ISODate("2013-01-17T00:08:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:08:09.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:08:10.654 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:08:12.654 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3471, "optime" : { "t" : 1358381292000, "i" : 8484 }, "optimeDate" : ISODate("2013-01-17T00:08:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3460, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3460, "optime" : { "t" : 1358381291000, "i" : 1291 }, "optimeDate" : ISODate("2013-01-17T00:08:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:12Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3470, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3457, "optime" : { "t" : 1358381288000, "i" : 7426 }, "optimeDate" : ISODate("2013-01-17T00:08:08Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:11Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3457, "optime" : { "t" : 1358381292000, "i" : 326 }, "optimeDate" : ISODate("2013-01-17T00:08:12Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3457, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3469, "optime" : { "t" : 1358381292000, "i" : 8460 }, "optimeDate" : ISODate("2013-01-17T00:08:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:08:13.967 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:08:14.654 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:08:16.654 [conn206] end connection 127.0.0.2:45212 (2 connections now open) m31001| Wed Jan 16 19:08:16.655 [initandlisten] connection accepted from 127.0.0.2:56901 #208 (3 connections now open) m31000| Wed Jan 16 19:08:16.655 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3476, "optime" : { "t" : 1358381297000, "i" : 1032 }, "optimeDate" : ISODate("2013-01-17T00:08:17Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3465, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3465, "optime" : { "t" : 1358381296000, "i" : 8335 }, "optimeDate" : ISODate("2013-01-17T00:08:16Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3475, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3462, "optime" : { "t" : 1358381292000, "i" : 8484 }, "optimeDate" : ISODate("2013-01-17T00:08:12Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:16Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3462, "optime" : { "t" : 1358381296000, "i" : 362 }, "optimeDate" : ISODate("2013-01-17T00:08:16Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3462, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3474, "optime" : { "t" : 1358381297000, "i" : 1032 }, "optimeDate" : ISODate("2013-01-17T00:08:17Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:08:18.090 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:08:18.655 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:08:19.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:08:20.091 [conn125] end connection 127.0.0.4:50634 (4 connections now open) m31000| Wed Jan 16 19:08:20.091 [initandlisten] connection accepted from 127.0.0.4:56307 #126 (5 connections now open) m31000| Wed Jan 16 19:08:20.655 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3480, "optime" : { "t" : 1358381301000, "i" : 2856 }, "optimeDate" : ISODate("2013-01-17T00:08:21Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3469, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3469, "optime" : { "t" : 1358381297000, "i" : 1032 }, "optimeDate" : ISODate("2013-01-17T00:08:17Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3479, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3466, "optime" : { "t" : 1358381301000, "i" : 1818 }, "optimeDate" : ISODate("2013-01-17T00:08:21Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3466, "optime" : { "t" : 1358381297000, "i" : 1032 }, "optimeDate" : ISODate("2013-01-17T00:08:17Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3466, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:21Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3478, "optime" : { "t" : 1358381301000, "i" : 2856 }, "optimeDate" : ISODate("2013-01-17T00:08:21Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:08:22.656 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:08:23.968 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:08:24.656 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3484, "optime" : { "t" : 1358381305000, "i" : 5106 }, "optimeDate" : ISODate("2013-01-17T00:08:25Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3473, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3473, "optime" : { "t" : 1358381304000, "i" : 74 }, "optimeDate" : ISODate("2013-01-17T00:08:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:07:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3483, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3470, "optime" : { "t" : 1358381305000, "i" : 2470 }, "optimeDate" : ISODate("2013-01-17T00:08:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:25Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3470, "optime" : { "t" : 1358381301000, "i" : 2856 }, "optimeDate" : ISODate("2013-01-17T00:08:21Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3470, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3482, "optime" : { "t" : 1358381305000, "i" : 5106 }, "optimeDate" : ISODate("2013-01-17T00:08:25Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:08:26.656 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:08:28.092 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:08:28.657 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3488, "optime" : { "t" : 1358381309000, "i" : 5580 }, "optimeDate" : ISODate("2013-01-17T00:08:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3477, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3477, "optime" : { "t" : 1358381309000, "i" : 4165 }, "optimeDate" : ISODate("2013-01-17T00:08:29Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3487, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3474, "optime" : { "t" : 1358381309000, "i" : 1698 }, "optimeDate" : ISODate("2013-01-17T00:08:29Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3474, "optime" : { "t" : 1358381305000, "i" : 5106 }, "optimeDate" : ISODate("2013-01-17T00:08:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:29Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3474, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:29Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3486, "optime" : { "t" : 1358381309000, "i" : 5580 }, "optimeDate" : ISODate("2013-01-17T00:08:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:08:30.657 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:08:31.976 [conn208] end connection 127.0.0.2:35010 (3 connections now open) m31002| Wed Jan 16 19:08:32.476 [initandlisten] connection accepted from 127.0.0.2:41143 #210 (4 connections now open) m31000| Wed Jan 16 19:08:32.657 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:08:33.248 [conn209] end connection 127.0.0.3:39576 (3 connections now open) m31002| Wed Jan 16 19:08:33.248 [initandlisten] connection accepted from 127.0.0.3:49659 #211 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3492, "optime" : { "t" : 1358381313000, "i" : 6861 }, "optimeDate" : ISODate("2013-01-17T00:08:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3481, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3481, "optime" : { "t" : 1358381309000, "i" : 5580 }, "optimeDate" : ISODate("2013-01-17T00:08:29Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3491, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3478, "optime" : { "t" : 1358381313000, "i" : 1943 }, "optimeDate" : ISODate("2013-01-17T00:08:33Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3478, "optime" : { "t" : 1358381309000, "i" : 5580 }, "optimeDate" : ISODate("2013-01-17T00:08:29Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3478, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:33Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3490, "optime" : { "t" : 1358381313000, "i" : 6861 }, "optimeDate" : ISODate("2013-01-17T00:08:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:08:33.969 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:08:34.658 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:08:36.469 [conn207] end connection 127.0.0.4:47947 (2 connections now open) m31000| Wed Jan 16 19:08:36.658 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:08:36.970 [initandlisten] connection accepted from 127.0.0.4:45454 #209 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3496, "optime" : { "t" : 1358381317000, "i" : 8611 }, "optimeDate" : ISODate("2013-01-17T00:08:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3485, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3485, "optime" : { "t" : 1358381317000, "i" : 4613 }, "optimeDate" : ISODate("2013-01-17T00:08:37Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3495, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3482, "optime" : { "t" : 1358381317000, "i" : 2704 }, "optimeDate" : ISODate("2013-01-17T00:08:37Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3482, "optime" : { "t" : 1358381313000, "i" : 6861 }, "optimeDate" : ISODate("2013-01-17T00:08:33Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:37Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3482, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3494, "optime" : { "t" : 1358381317000, "i" : 8611 }, "optimeDate" : ISODate("2013-01-17T00:08:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:08:38.094 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:08:38.658 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:08:40.659 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:08:41.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3501, "optime" : { "t" : 1358381322000, "i" : 1191 }, "optimeDate" : ISODate("2013-01-17T00:08:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3490, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3490, "optime" : { "t" : 1358381317000, "i" : 8611 }, "optimeDate" : ISODate("2013-01-17T00:08:37Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:42Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3500, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3487, "optime" : { "t" : 1358381321000, "i" : 1974 }, "optimeDate" : ISODate("2013-01-17T00:08:41Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:41Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3487, "optime" : { "t" : 1358381322000, "i" : 1157 }, "optimeDate" : ISODate("2013-01-17T00:08:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3487, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3499, "optime" : { "t" : 1358381322000, "i" : 1160 }, "optimeDate" : ISODate("2013-01-17T00:08:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:08:42.659 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:08:44.095 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:08:44.660 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3505, "optime" : { "t" : 1358381326000, "i" : 2982 }, "optimeDate" : ISODate("2013-01-17T00:08:46Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3494, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3494, "optime" : { "t" : 1358381322000, "i" : 1191 }, "optimeDate" : ISODate("2013-01-17T00:08:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:46Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3504, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3491, "optime" : { "t" : 1358381325000, "i" : 920 }, "optimeDate" : ISODate("2013-01-17T00:08:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3491, "optime" : { "t" : 1358381326000, "i" : 1243 }, "optimeDate" : ISODate("2013-01-17T00:08:46Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3491, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3503, "optime" : { "t" : 1358381326000, "i" : 2982 }, "optimeDate" : ISODate("2013-01-17T00:08:46Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:08:46.660 [conn208] end connection 127.0.0.2:56901 (2 connections now open) m31001| Wed Jan 16 19:08:46.660 [initandlisten] connection accepted from 127.0.0.2:41596 #210 (3 connections now open) m31000| Wed Jan 16 19:08:46.660 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:08:48.661 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:08:49.472 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:08:50.096 [conn126] end connection 127.0.0.4:56307 (4 connections now open) m31000| Wed Jan 16 19:08:50.096 [initandlisten] connection accepted from 127.0.0.4:33034 #127 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3509, "optime" : { "t" : 1358381330000, "i" : 4262 }, "optimeDate" : ISODate("2013-01-17T00:08:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3498, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3498, "optime" : { "t" : 1358381329000, "i" : 5344 }, "optimeDate" : ISODate("2013-01-17T00:08:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:50Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3508, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3495, "optime" : { "t" : 1358381326000, "i" : 2982 }, "optimeDate" : ISODate("2013-01-17T00:08:46Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:49Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3495, "optime" : { "t" : 1358381330000, "i" : 469 }, "optimeDate" : ISODate("2013-01-17T00:08:50Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3495, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3507, "optime" : { "t" : 1358381330000, "i" : 4262 }, "optimeDate" : ISODate("2013-01-17T00:08:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:08:50.661 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:08:51.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:08:52.661 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:08:54.097 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:08:54.662 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3513, "optime" : { "t" : 1358381334000, "i" : 6139 }, "optimeDate" : ISODate("2013-01-17T00:08:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3502, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3502, "optime" : { "t" : 1358381330000, "i" : 4262 }, "optimeDate" : ISODate("2013-01-17T00:08:50Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:54Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3512, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3499, "optime" : { "t" : 1358381330000, "i" : 4262 }, "optimeDate" : ISODate("2013-01-17T00:08:50Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:54Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3499, "optime" : { "t" : 1358381334000, "i" : 608 }, "optimeDate" : ISODate("2013-01-17T00:08:54Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3499, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3511, "optime" : { "t" : 1358381334000, "i" : 6084 }, "optimeDate" : ISODate("2013-01-17T00:08:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:08:56.662 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:08:58.662 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3517, "optime" : { "t" : 1358381338000, "i" : 7366 }, "optimeDate" : ISODate("2013-01-17T00:08:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3506, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3506, "optime" : { "t" : 1358381334000, "i" : 6139 }, "optimeDate" : ISODate("2013-01-17T00:08:54Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:58Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3516, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3503, "optime" : { "t" : 1358381334000, "i" : 6139 }, "optimeDate" : ISODate("2013-01-17T00:08:54Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:08:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3503, "optime" : { "t" : 1358381338000, "i" : 425 }, "optimeDate" : ISODate("2013-01-17T00:08:58Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3503, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:08:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:08:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3515, "optime" : { "t" : 1358381338000, "i" : 7107 }, "optimeDate" : ISODate("2013-01-17T00:08:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:08:59.473 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:09:00.663 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:09:02.663 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3522, "optime" : { "t" : 1358381343000, "i" : 284 }, "optimeDate" : ISODate("2013-01-17T00:09:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3511, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3511, "optime" : { "t" : 1358381342000, "i" : 3790 }, "optimeDate" : ISODate("2013-01-17T00:09:02Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3521, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3508, "optime" : { "t" : 1358381338000, "i" : 7366 }, "optimeDate" : ISODate("2013-01-17T00:08:58Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:01Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3508, "optime" : { "t" : 1358381342000, "i" : 856 }, "optimeDate" : ISODate("2013-01-17T00:09:02Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3508, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3520, "optime" : { "t" : 1358381343000, "i" : 284 }, "optimeDate" : ISODate("2013-01-17T00:09:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:09:03.254 [conn211] end connection 127.0.0.3:49659 (3 connections now open) m31002| Wed Jan 16 19:09:03.254 [initandlisten] connection accepted from 127.0.0.3:49468 #212 (4 connections now open) m31002| Wed Jan 16 19:09:04.099 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:09:04.664 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:09:06.664 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3526, "optime" : { "t" : 1358381347000, "i" : 1231 }, "optimeDate" : ISODate("2013-01-17T00:09:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3515, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3515, "optime" : { "t" : 1358381343000, "i" : 284 }, "optimeDate" : ISODate("2013-01-17T00:09:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:06Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3525, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3512, "optime" : { "t" : 1358381343000, "i" : 284 }, "optimeDate" : ISODate("2013-01-17T00:09:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:06Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3512, "optime" : { "t" : 1358381346000, "i" : 365 }, "optimeDate" : ISODate("2013-01-17T00:09:06Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3512, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3524, "optime" : { "t" : 1358381347000, "i" : 934 }, "optimeDate" : ISODate("2013-01-17T00:09:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:09:08.664 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:09:09.474 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 19:09:09.981 [conn210] end connection 127.0.0.2:41143 (3 connections now open) m31002| Wed Jan 16 19:09:10.481 [initandlisten] connection accepted from 127.0.0.2:55587 #213 (4 connections now open) m31000| Wed Jan 16 19:09:10.664 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3530, "optime" : { "t" : 1358381351000, "i" : 2777 }, "optimeDate" : ISODate("2013-01-17T00:09:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3519, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3519, "optime" : { "t" : 1358381350000, "i" : 2243 }, "optimeDate" : ISODate("2013-01-17T00:09:10Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3529, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3516, "optime" : { "t" : 1358381351000, "i" : 2301 }, "optimeDate" : ISODate("2013-01-17T00:09:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3516, "optime" : { "t" : 1358381347000, "i" : 1231 }, "optimeDate" : ISODate("2013-01-17T00:09:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3516, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3528, "optime" : { "t" : 1358381351000, "i" : 2745 }, "optimeDate" : ISODate("2013-01-17T00:09:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:09:12.665 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:09:13.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:09:14.100 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:09:14.475 [conn209] end connection 127.0.0.4:45454 (2 connections now open) m31000| Wed Jan 16 19:09:14.665 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:09:14.975 [initandlisten] connection accepted from 127.0.0.4:33834 #211 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3534, "optime" : { "t" : 1358381355000, "i" : 4288 }, "optimeDate" : ISODate("2013-01-17T00:09:15Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3523, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3523, "optime" : { "t" : 1358381355000, "i" : 4134 }, "optimeDate" : ISODate("2013-01-17T00:09:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3533, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3520, "optime" : { "t" : 1358381355000, "i" : 2584 }, "optimeDate" : ISODate("2013-01-17T00:09:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3520, "optime" : { "t" : 1358381351000, "i" : 2777 }, "optimeDate" : ISODate("2013-01-17T00:09:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:15Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3520, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:15Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3532, "optime" : { "t" : 1358381355000, "i" : 4288 }, "optimeDate" : ISODate("2013-01-17T00:09:15Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:09:16.665 [conn210] end connection 127.0.0.2:41596 (2 connections now open) m31001| Wed Jan 16 19:09:16.666 [initandlisten] connection accepted from 127.0.0.2:51139 #212 (3 connections now open) m31000| Wed Jan 16 19:09:16.666 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:09:18.666 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3538, "optime" : { "t" : 1358381359000, "i" : 5746 }, "optimeDate" : ISODate("2013-01-17T00:09:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3527, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3527, "optime" : { "t" : 1358381355000, "i" : 4288 }, "optimeDate" : ISODate("2013-01-17T00:09:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:18Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3537, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3524, "optime" : { "t" : 1358381359000, "i" : 2409 }, "optimeDate" : ISODate("2013-01-17T00:09:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3524, "optime" : { "t" : 1358381355000, "i" : 4288 }, "optimeDate" : ISODate("2013-01-17T00:09:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3524, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:19Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3536, "optime" : { "t" : 1358381359000, "i" : 5182 }, "optimeDate" : ISODate("2013-01-17T00:09:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:09:19.976 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:09:20.101 [conn127] end connection 127.0.0.4:33034 (4 connections now open) m31000| Wed Jan 16 19:09:20.101 [initandlisten] connection accepted from 127.0.0.4:47611 #128 (5 connections now open) m31000| Wed Jan 16 19:09:20.666 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:09:22.667 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:09:23.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 19:09:23.913 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:593739 296ms { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3542, "optime" : { "t" : 1358381363000, "i" : 7213 }, "optimeDate" : ISODate("2013-01-17T00:09:23Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3531, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3531, "optime" : { "t" : 1358381362000, "i" : 2654 }, "optimeDate" : ISODate("2013-01-17T00:09:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3541, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3528, "optime" : { "t" : 1358381363000, "i" : 2496 }, "optimeDate" : ISODate("2013-01-17T00:09:23Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3528, "optime" : { "t" : 1358381359000, "i" : 5746 }, "optimeDate" : ISODate("2013-01-17T00:09:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3528, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3540, "optime" : { "t" : 1358381363000, "i" : 7171 }, "optimeDate" : ISODate("2013-01-17T00:09:23Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:09:24.103 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:09:24.667 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:09:26.668 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3547, "optime" : { "t" : 1358381368000, "i" : 680 }, "optimeDate" : ISODate("2013-01-17T00:09:28Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3536, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3536, "optime" : { "t" : 1358381367000, "i" : 9258 }, "optimeDate" : ISODate("2013-01-17T00:09:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3546, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3533, "optime" : { "t" : 1358381367000, "i" : 2637 }, "optimeDate" : ISODate("2013-01-17T00:09:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:27Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:28Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3533, "optime" : { "t" : 1358381363000, "i" : 7213 }, "optimeDate" : ISODate("2013-01-17T00:09:23Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:27Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3533, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3545, "optime" : { "t" : 1358381367000, "i" : 9309 }, "optimeDate" : ISODate("2013-01-17T00:09:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:09:28.668 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:09:29.977 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:09:30.668 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3551, "optime" : { "t" : 1358381372000, "i" : 2201 }, "optimeDate" : ISODate("2013-01-17T00:09:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3540, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3540, "optime" : { "t" : 1358381368000, "i" : 680 }, "optimeDate" : ISODate("2013-01-17T00:09:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:32Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3550, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3537, "optime" : { "t" : 1358381371000, "i" : 1386 }, "optimeDate" : ISODate("2013-01-17T00:09:31Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3537, "optime" : { "t" : 1358381372000, "i" : 866 }, "optimeDate" : ISODate("2013-01-17T00:09:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3537, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:31Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3549, "optime" : { "t" : 1358381372000, "i" : 2201 }, "optimeDate" : ISODate("2013-01-17T00:09:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:09:32.669 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:09:33.259 [conn212] end connection 127.0.0.3:49468 (3 connections now open) m31002| Wed Jan 16 19:09:33.259 [initandlisten] connection accepted from 127.0.0.3:45069 #214 (4 connections now open) m31002| Wed Jan 16 19:09:34.104 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:09:34.669 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3555, "optime" : { "t" : 1358381376000, "i" : 3493 }, "optimeDate" : ISODate("2013-01-17T00:09:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3544, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3544, "optime" : { "t" : 1358381375000, "i" : 1761 }, "optimeDate" : ISODate("2013-01-17T00:09:35Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:36Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3554, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3541, "optime" : { "t" : 1358381372000, "i" : 2201 }, "optimeDate" : ISODate("2013-01-17T00:09:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3541, "optime" : { "t" : 1358381376000, "i" : 603 }, "optimeDate" : ISODate("2013-01-17T00:09:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3541, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:35Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3553, "optime" : { "t" : 1358381376000, "i" : 3493 }, "optimeDate" : ISODate("2013-01-17T00:09:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:09:36.670 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:09:38.670 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:09:39.978 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3559, "optime" : { "t" : 1358381380000, "i" : 5149 }, "optimeDate" : ISODate("2013-01-17T00:09:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3548, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3548, "optime" : { "t" : 1358381380000, "i" : 4387 }, "optimeDate" : ISODate("2013-01-17T00:09:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3558, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3545, "optime" : { "t" : 1358381376000, "i" : 3493 }, "optimeDate" : ISODate("2013-01-17T00:09:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:39Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3545, "optime" : { "t" : 1358381380000, "i" : 1008 }, "optimeDate" : ISODate("2013-01-17T00:09:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3545, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3557, "optime" : { "t" : 1358381380000, "i" : 5149 }, "optimeDate" : ISODate("2013-01-17T00:09:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:09:40.670 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:09:42.671 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:09:44.106 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:09:44.671 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3563, "optime" : { "t" : 1358381384000, "i" : 7242 }, "optimeDate" : ISODate("2013-01-17T00:09:44Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3552, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3552, "optime" : { "t" : 1358381380000, "i" : 5149 }, "optimeDate" : ISODate("2013-01-17T00:09:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:44Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3562, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3549, "optime" : { "t" : 1358381380000, "i" : 5149 }, "optimeDate" : ISODate("2013-01-17T00:09:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3549, "optime" : { "t" : 1358381384000, "i" : 1345 }, "optimeDate" : ISODate("2013-01-17T00:09:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3549, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:43Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3561, "optime" : { "t" : 1358381384000, "i" : 7084 }, "optimeDate" : ISODate("2013-01-17T00:09:44Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:09:45.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 19:09:46.671 [conn212] end connection 127.0.0.2:51139 (2 connections now open) m31001| Wed Jan 16 19:09:46.672 [initandlisten] connection accepted from 127.0.0.2:53756 #213 (3 connections now open) m31000| Wed Jan 16 19:09:46.672 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:09:47.986 [conn213] end connection 127.0.0.2:55587 (3 connections now open) m31002| Wed Jan 16 19:09:48.486 [initandlisten] connection accepted from 127.0.0.2:60434 #215 (4 connections now open) m31000| Wed Jan 16 19:09:48.672 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3567, "optime" : { "t" : 1358381388000, "i" : 8146 }, "optimeDate" : ISODate("2013-01-17T00:09:48Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3556, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3556, "optime" : { "t" : 1358381388000, "i" : 3914 }, "optimeDate" : ISODate("2013-01-17T00:09:48Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3566, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3553, "optime" : { "t" : 1358381384000, "i" : 7242 }, "optimeDate" : ISODate("2013-01-17T00:09:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:47Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3553, "optime" : { "t" : 1358381388000, "i" : 1150 }, "optimeDate" : ISODate("2013-01-17T00:09:48Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3553, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3565, "optime" : { "t" : 1358381388000, "i" : 8146 }, "optimeDate" : ISODate("2013-01-17T00:09:48Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:09:49.980 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:09:50.106 [conn128] end connection 127.0.0.4:47611 (4 connections now open) m31000| Wed Jan 16 19:09:50.107 [initandlisten] connection accepted from 127.0.0.4:57833 #129 (5 connections now open) m31000| Wed Jan 16 19:09:50.673 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:09:52.480 [conn211] end connection 127.0.0.4:33834 (2 connections now open) m31000| Wed Jan 16 19:09:52.673 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:09:52.980 [initandlisten] connection accepted from 127.0.0.4:33702 #214 (3 connections now open) STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 19:09:53.165 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:310996 155ms { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3572, "optime" : { "t" : 1358381393000, "i" : 603 }, "optimeDate" : ISODate("2013-01-17T00:09:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3561, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3561, "optime" : { "t" : 1358381388000, "i" : 8146 }, "optimeDate" : ISODate("2013-01-17T00:09:48Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:52Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3571, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3558, "optime" : { "t" : 1358381388000, "i" : 8146 }, "optimeDate" : ISODate("2013-01-17T00:09:48Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:52Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3558, "optime" : { "t" : 1358381392000, "i" : 1018 }, "optimeDate" : ISODate("2013-01-17T00:09:52Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3558, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3570, "optime" : { "t" : 1358381393000, "i" : 42 }, "optimeDate" : ISODate("2013-01-17T00:09:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:09:54.108 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:09:54.673 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:09:55.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:09:56.674 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3576, "optime" : { "t" : 1358381397000, "i" : 3000 }, "optimeDate" : ISODate("2013-01-17T00:09:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3565, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3565, "optime" : { "t" : 1358381393000, "i" : 603 }, "optimeDate" : ISODate("2013-01-17T00:09:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:56Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3575, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3562, "optime" : { "t" : 1358381397000, "i" : 1937 }, "optimeDate" : ISODate("2013-01-17T00:09:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:09:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3562, "optime" : { "t" : 1358381393000, "i" : 603 }, "optimeDate" : ISODate("2013-01-17T00:09:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3562, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:09:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3574, "optime" : { "t" : 1358381397000, "i" : 3000 }, "optimeDate" : ISODate("2013-01-17T00:09:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:09:58.674 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:10:00.109 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:10:00.674 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3580, "optime" : { "t" : 1358381401000, "i" : 4787 }, "optimeDate" : ISODate("2013-01-17T00:10:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3569, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3569, "optime" : { "t" : 1358381400000, "i" : 4301 }, "optimeDate" : ISODate("2013-01-17T00:10:00Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3579, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3566, "optime" : { "t" : 1358381401000, "i" : 1607 }, "optimeDate" : ISODate("2013-01-17T00:10:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3566, "optime" : { "t" : 1358381397000, "i" : 3000 }, "optimeDate" : ISODate("2013-01-17T00:09:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3566, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:09:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3578, "optime" : { "t" : 1358381401000, "i" : 4349 }, "optimeDate" : ISODate("2013-01-17T00:10:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:10:02.675 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:10:03.265 [conn214] end connection 127.0.0.3:45069 (3 connections now open) m31002| Wed Jan 16 19:10:03.265 [initandlisten] connection accepted from 127.0.0.3:35699 #216 (4 connections now open) m31000| Wed Jan 16 19:10:04.675 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:10:05.482 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3584, "optime" : { "t" : 1358381405000, "i" : 6754 }, "optimeDate" : ISODate("2013-01-17T00:10:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3573, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3573, "optime" : { "t" : 1358381401000, "i" : 4787 }, "optimeDate" : ISODate("2013-01-17T00:10:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:04Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3583, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3570, "optime" : { "t" : 1358381405000, "i" : 2733 }, "optimeDate" : ISODate("2013-01-17T00:10:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:05Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3570, "optime" : { "t" : 1358381401000, "i" : 4787 }, "optimeDate" : ISODate("2013-01-17T00:10:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3570, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3582, "optime" : { "t" : 1358381405000, "i" : 6754 }, "optimeDate" : ISODate("2013-01-17T00:10:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:10:06.675 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:10:08.676 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3588, "optime" : { "t" : 1358381409000, "i" : 8011 }, "optimeDate" : ISODate("2013-01-17T00:10:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3577, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3577, "optime" : { "t" : 1358381405000, "i" : 6754 }, "optimeDate" : ISODate("2013-01-17T00:10:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3587, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3574, "optime" : { "t" : 1358381409000, "i" : 2373 }, "optimeDate" : ISODate("2013-01-17T00:10:09Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3574, "optime" : { "t" : 1358381405000, "i" : 6754 }, "optimeDate" : ISODate("2013-01-17T00:10:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3574, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:09Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3586, "optime" : { "t" : 1358381409000, "i" : 8011 }, "optimeDate" : ISODate("2013-01-17T00:10:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:10:10.111 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:10:10.676 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:10:12.676 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3593, "optime" : { "t" : 1358381414000, "i" : 301 }, "optimeDate" : ISODate("2013-01-17T00:10:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3582, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3582, "optime" : { "t" : 1358381413000, "i" : 3821 }, "optimeDate" : ISODate("2013-01-17T00:10:13Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3592, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3579, "optime" : { "t" : 1358381413000, "i" : 2293 }, "optimeDate" : ISODate("2013-01-17T00:10:13Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3579, "optime" : { "t" : 1358381409000, "i" : 8011 }, "optimeDate" : ISODate("2013-01-17T00:10:09Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3579, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:13Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3591, "optime" : { "t" : 1358381414000, "i" : 301 }, "optimeDate" : ISODate("2013-01-17T00:10:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:10:14.677 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:10:15.484 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:10:16.677 [conn213] end connection 127.0.0.2:53756 (2 connections now open) m31001| Wed Jan 16 19:10:16.677 [initandlisten] connection accepted from 127.0.0.2:52111 #215 (3 connections now open) m31000| Wed Jan 16 19:10:16.678 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:10:17.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3597, "optime" : { "t" : 1358381418000, "i" : 2158 }, "optimeDate" : ISODate("2013-01-17T00:10:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3586, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3586, "optime" : { "t" : 1358381414000, "i" : 301 }, "optimeDate" : ISODate("2013-01-17T00:10:14Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:18Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3596, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3583, "optime" : { "t" : 1358381417000, "i" : 1794 }, "optimeDate" : ISODate("2013-01-17T00:10:17Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:17Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3583, "optime" : { "t" : 1358381418000, "i" : 1239 }, "optimeDate" : ISODate("2013-01-17T00:10:18Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3583, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3595, "optime" : { "t" : 1358381418000, "i" : 2158 }, "optimeDate" : ISODate("2013-01-17T00:10:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:10:18.678 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:10:20.112 [conn129] end connection 127.0.0.4:57833 (4 connections now open) m31000| Wed Jan 16 19:10:20.112 [initandlisten] connection accepted from 127.0.0.4:47885 #130 (5 connections now open) m31002| Wed Jan 16 19:10:20.113 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:10:20.678 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3601, "optime" : { "t" : 1358381422000, "i" : 2976 }, "optimeDate" : ISODate("2013-01-17T00:10:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3590, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3590, "optime" : { "t" : 1358381418000, "i" : 2158 }, "optimeDate" : ISODate("2013-01-17T00:10:18Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:22Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3600, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3587, "optime" : { "t" : 1358381421000, "i" : 392 }, "optimeDate" : ISODate("2013-01-17T00:10:21Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3587, "optime" : { "t" : 1358381422000, "i" : 855 }, "optimeDate" : ISODate("2013-01-17T00:10:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3587, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:21Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3599, "optime" : { "t" : 1358381422000, "i" : 2976 }, "optimeDate" : ISODate("2013-01-17T00:10:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:10:22.679 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:10:24.679 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:10:25.485 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 19:10:25.991 [conn215] end connection 127.0.0.2:60434 (3 connections now open) m31002| Wed Jan 16 19:10:26.491 [initandlisten] connection accepted from 127.0.0.2:43939 #217 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3605, "optime" : { "t" : 1358381426000, "i" : 4985 }, "optimeDate" : ISODate("2013-01-17T00:10:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3594, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3594, "optime" : { "t" : 1358381426000, "i" : 3987 }, "optimeDate" : ISODate("2013-01-17T00:10:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3604, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3591, "optime" : { "t" : 1358381422000, "i" : 2976 }, "optimeDate" : ISODate("2013-01-17T00:10:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:25Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3591, "optime" : { "t" : 1358381426000, "i" : 782 }, "optimeDate" : ISODate("2013-01-17T00:10:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3591, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3603, "optime" : { "t" : 1358381426000, "i" : 4985 }, "optimeDate" : ISODate("2013-01-17T00:10:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:10:26.679 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:10:27.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:10:28.680 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:10:30.114 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:10:30.485 [conn214] end connection 127.0.0.4:33702 (2 connections now open) m31000| Wed Jan 16 19:10:30.680 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3609, "optime" : { "t" : 1358381430000, "i" : 6679 }, "optimeDate" : ISODate("2013-01-17T00:10:30Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3598, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3598, "optime" : { "t" : 1358381426000, "i" : 4985 }, "optimeDate" : ISODate("2013-01-17T00:10:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:30Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3608, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3595, "optime" : { "t" : 1358381426000, "i" : 4985 }, "optimeDate" : ISODate("2013-01-17T00:10:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3595, "optime" : { "t" : 1358381430000, "i" : 1175 }, "optimeDate" : ISODate("2013-01-17T00:10:30Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3595, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:29Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3607, "optime" : { "t" : 1358381430000, "i" : 6611 }, "optimeDate" : ISODate("2013-01-17T00:10:30Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:10:30.985 [initandlisten] connection accepted from 127.0.0.4:42413 #216 (3 connections now open) m31000| Wed Jan 16 19:10:32.681 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:10:33.270 [conn216] end connection 127.0.0.3:35699 (3 connections now open) m31002| Wed Jan 16 19:10:33.270 [initandlisten] connection accepted from 127.0.0.3:33137 #218 (4 connections now open) m31000| Wed Jan 16 19:10:34.042 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:372080 186ms m31000| Wed Jan 16 19:10:34.681 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3613, "optime" : { "t" : 1358381434000, "i" : 9674 }, "optimeDate" : ISODate("2013-01-17T00:10:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3602, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3602, "optime" : { "t" : 1358381433000, "i" : 203 }, "optimeDate" : ISODate("2013-01-17T00:10:33Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:34Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3612, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3599, "optime" : { "t" : 1358381430000, "i" : 6679 }, "optimeDate" : ISODate("2013-01-17T00:10:30Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:33Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3599, "optime" : { "t" : 1358381434000, "i" : 2204 }, "optimeDate" : ISODate("2013-01-17T00:10:34Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3599, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3611, "optime" : { "t" : 1358381434000, "i" : 9674 }, "optimeDate" : ISODate("2013-01-17T00:10:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:10:35.987 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:10:36.681 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:10:38.682 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3618, "optime" : { "t" : 1358381439000, "i" : 576 }, "optimeDate" : ISODate("2013-01-17T00:10:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3607, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3607, "optime" : { "t" : 1358381438000, "i" : 8507 }, "optimeDate" : ISODate("2013-01-17T00:10:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3617, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3604, "optime" : { "t" : 1358381434000, "i" : 9674 }, "optimeDate" : ISODate("2013-01-17T00:10:34Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:38Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3604, "optime" : { "t" : 1358381438000, "i" : 656 }, "optimeDate" : ISODate("2013-01-17T00:10:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3604, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3616, "optime" : { "t" : 1358381439000, "i" : 576 }, "optimeDate" : ISODate("2013-01-17T00:10:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:10:40.116 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:10:40.682 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:10:42.682 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3622, "optime" : { "t" : 1358381443000, "i" : 2147 }, "optimeDate" : ISODate("2013-01-17T00:10:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3611, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3611, "optime" : { "t" : 1358381439000, "i" : 576 }, "optimeDate" : ISODate("2013-01-17T00:10:39Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:42Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3621, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3608, "optime" : { "t" : 1358381439000, "i" : 576 }, "optimeDate" : ISODate("2013-01-17T00:10:39Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3608, "optime" : { "t" : 1358381442000, "i" : 220 }, "optimeDate" : ISODate("2013-01-17T00:10:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3608, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:41Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3620, "optime" : { "t" : 1358381443000, "i" : 2147 }, "optimeDate" : ISODate("2013-01-17T00:10:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:10:44.683 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:10:45.988 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:10:46.683 [conn215] end connection 127.0.0.2:52111 (2 connections now open) m31001| Wed Jan 16 19:10:46.683 [initandlisten] connection accepted from 127.0.0.2:52043 #217 (3 connections now open) m31000| Wed Jan 16 19:10:46.683 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3626, "optime" : { "t" : 1358381447000, "i" : 3236 }, "optimeDate" : ISODate("2013-01-17T00:10:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3615, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3615, "optime" : { "t" : 1358381446000, "i" : 2337 }, "optimeDate" : ISODate("2013-01-17T00:10:46Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3625, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3612, "optime" : { "t" : 1358381447000, "i" : 2298 }, "optimeDate" : ISODate("2013-01-17T00:10:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3612, "optime" : { "t" : 1358381443000, "i" : 2147 }, "optimeDate" : ISODate("2013-01-17T00:10:43Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3612, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3624, "optime" : { "t" : 1358381447000, "i" : 3236 }, "optimeDate" : ISODate("2013-01-17T00:10:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:10:48.684 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:10:49.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:10:50.117 [conn130] end connection 127.0.0.4:47885 (4 connections now open) m31000| Wed Jan 16 19:10:50.117 [initandlisten] connection accepted from 127.0.0.4:59174 #131 (5 connections now open) m31002| Wed Jan 16 19:10:50.118 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:10:50.684 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3630, "optime" : { "t" : 1358381451000, "i" : 4416 }, "optimeDate" : ISODate("2013-01-17T00:10:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3619, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3619, "optime" : { "t" : 1358381447000, "i" : 3236 }, "optimeDate" : ISODate("2013-01-17T00:10:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:50Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3629, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3616, "optime" : { "t" : 1358381451000, "i" : 2846 }, "optimeDate" : ISODate("2013-01-17T00:10:51Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3616, "optime" : { "t" : 1358381447000, "i" : 3236 }, "optimeDate" : ISODate("2013-01-17T00:10:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3616, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3628, "optime" : { "t" : 1358381451000, "i" : 4340 }, "optimeDate" : ISODate("2013-01-17T00:10:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:10:52.684 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:10:54.685 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3634, "optime" : { "t" : 1358381455000, "i" : 5313 }, "optimeDate" : ISODate("2013-01-17T00:10:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3623, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3623, "optime" : { "t" : 1358381451000, "i" : 4416 }, "optimeDate" : ISODate("2013-01-17T00:10:51Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:54Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3633, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3620, "optime" : { "t" : 1358381455000, "i" : 2071 }, "optimeDate" : ISODate("2013-01-17T00:10:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3620, "optime" : { "t" : 1358381451000, "i" : 4416 }, "optimeDate" : ISODate("2013-01-17T00:10:51Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3620, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3632, "optime" : { "t" : 1358381455000, "i" : 4986 }, "optimeDate" : ISODate("2013-01-17T00:10:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:10:55.989 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:10:56.685 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:10:58.685 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:10:59.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3638, "optime" : { "t" : 1358381459000, "i" : 6953 }, "optimeDate" : ISODate("2013-01-17T00:10:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3627, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3627, "optime" : { "t" : 1358381458000, "i" : 3039 }, "optimeDate" : ISODate("2013-01-17T00:10:58Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3637, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3624, "optime" : { "t" : 1358381459000, "i" : 2647 }, "optimeDate" : ISODate("2013-01-17T00:10:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:10:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3624, "optime" : { "t" : 1358381455000, "i" : 5313 }, "optimeDate" : ISODate("2013-01-17T00:10:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3624, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:10:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3636, "optime" : { "t" : 1358381459000, "i" : 6953 }, "optimeDate" : ISODate("2013-01-17T00:10:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:11:00.119 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:11:00.686 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:11:02.686 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:11:03.276 [conn218] end connection 127.0.0.3:33137 (3 connections now open) m31002| Wed Jan 16 19:11:03.276 [initandlisten] connection accepted from 127.0.0.3:53951 #219 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3642, "optime" : { "t" : 1358381463000, "i" : 8466 }, "optimeDate" : ISODate("2013-01-17T00:11:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3631, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3631, "optime" : { "t" : 1358381459000, "i" : 6953 }, "optimeDate" : ISODate("2013-01-17T00:10:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:02Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3641, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3628, "optime" : { "t" : 1358381463000, "i" : 2707 }, "optimeDate" : ISODate("2013-01-17T00:11:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:03Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3628, "optime" : { "t" : 1358381459000, "i" : 6953 }, "optimeDate" : ISODate("2013-01-17T00:10:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3628, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3640, "optime" : { "t" : 1358381463000, "i" : 8365 }, "optimeDate" : ISODate("2013-01-17T00:11:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:11:03.997 [conn217] end connection 127.0.0.2:43939 (3 connections now open) m31002| Wed Jan 16 19:11:04.497 [initandlisten] connection accepted from 127.0.0.2:38162 #220 (4 connections now open) m31000| Wed Jan 16 19:11:04.687 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:11:05.990 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:11:06.687 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3647, "optime" : { "t" : 1358381468000, "i" : 208 }, "optimeDate" : ISODate("2013-01-17T00:11:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3636, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3636, "optime" : { "t" : 1358381466000, "i" : 380 }, "optimeDate" : ISODate("2013-01-17T00:11:06Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3646, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3633, "optime" : { "t" : 1358381467000, "i" : 2358 }, "optimeDate" : ISODate("2013-01-17T00:11:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3633, "optime" : { "t" : 1358381463000, "i" : 8466 }, "optimeDate" : ISODate("2013-01-17T00:11:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3633, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:07Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3645, "optime" : { "t" : 1358381468000, "i" : 141 }, "optimeDate" : ISODate("2013-01-17T00:11:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:11:08.491 [conn216] end connection 127.0.0.4:42413 (2 connections now open) m31000| Wed Jan 16 19:11:08.687 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:11:08.991 [initandlisten] connection accepted from 127.0.0.4:34868 #218 (3 connections now open) m31001| Wed Jan 16 19:11:09.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:11:10.121 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:11:10.688 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3651, "optime" : { "t" : 1358381472000, "i" : 1370 }, "optimeDate" : ISODate("2013-01-17T00:11:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3640, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3640, "optime" : { "t" : 1358381471000, "i" : 8293 }, "optimeDate" : ISODate("2013-01-17T00:11:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:12Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3650, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3637, "optime" : { "t" : 1358381471000, "i" : 2155 }, "optimeDate" : ISODate("2013-01-17T00:11:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:11Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3637, "optime" : { "t" : 1358381472000, "i" : 808 }, "optimeDate" : ISODate("2013-01-17T00:11:12Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3637, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3649, "optime" : { "t" : 1358381472000, "i" : 1194 }, "optimeDate" : ISODate("2013-01-17T00:11:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:11:12.688 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:11:14.689 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:11:16.122 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3655, "optime" : { "t" : 1358381476000, "i" : 2170 }, "optimeDate" : ISODate("2013-01-17T00:11:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3644, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3644, "optime" : { "t" : 1358381472000, "i" : 1370 }, "optimeDate" : ISODate("2013-01-17T00:11:12Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:10:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3654, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3641, "optime" : { "t" : 1358381475000, "i" : 1158 }, "optimeDate" : ISODate("2013-01-17T00:11:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3641, "optime" : { "t" : 1358381476000, "i" : 1117 }, "optimeDate" : ISODate("2013-01-17T00:11:16Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3641, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:15Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3653, "optime" : { "t" : 1358381476000, "i" : 2170 }, "optimeDate" : ISODate("2013-01-17T00:11:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:11:16.689 [conn217] end connection 127.0.0.2:52043 (2 connections now open) m31001| Wed Jan 16 19:11:16.689 [initandlisten] connection accepted from 127.0.0.2:40980 #219 (3 connections now open) m31000| Wed Jan 16 19:11:16.689 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:11:18.690 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:11:20.122 [conn131] end connection 127.0.0.4:59174 (4 connections now open) m31000| Wed Jan 16 19:11:20.122 [initandlisten] connection accepted from 127.0.0.4:35259 #132 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3659, "optime" : { "t" : 1358381480000, "i" : 3967 }, "optimeDate" : ISODate("2013-01-17T00:11:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3648, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3648, "optime" : { "t" : 1358381479000, "i" : 1973 }, "optimeDate" : ISODate("2013-01-17T00:11:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3658, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3645, "optime" : { "t" : 1358381479000, "i" : 11 }, "optimeDate" : ISODate("2013-01-17T00:11:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3645, "optime" : { "t" : 1358381480000, "i" : 1145 }, "optimeDate" : ISODate("2013-01-17T00:11:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3645, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:19Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3657, "optime" : { "t" : 1358381480000, "i" : 3967 }, "optimeDate" : ISODate("2013-01-17T00:11:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:11:20.690 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:11:21.493 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:11:22.690 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3663, "optime" : { "t" : 1358381484000, "i" : 5157 }, "optimeDate" : ISODate("2013-01-17T00:11:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3652, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3652, "optime" : { "t" : 1358381484000, "i" : 4680 }, "optimeDate" : ISODate("2013-01-17T00:11:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3662, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3649, "optime" : { "t" : 1358381480000, "i" : 3967 }, "optimeDate" : ISODate("2013-01-17T00:11:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:23Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3649, "optime" : { "t" : 1358381484000, "i" : 1407 }, "optimeDate" : ISODate("2013-01-17T00:11:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3649, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3661, "optime" : { "t" : 1358381484000, "i" : 4683 }, "optimeDate" : ISODate("2013-01-17T00:11:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:11:24.691 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:11:26.123 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:11:26.691 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:11:28.691 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3667, "optime" : { "t" : 1358381488000, "i" : 6661 }, "optimeDate" : ISODate("2013-01-17T00:11:28Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3656, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3656, "optime" : { "t" : 1358381484000, "i" : 5157 }, "optimeDate" : ISODate("2013-01-17T00:11:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:28Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3666, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3653, "optime" : { "t" : 1358381484000, "i" : 5157 }, "optimeDate" : ISODate("2013-01-17T00:11:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:28Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3653, "optime" : { "t" : 1358381488000, "i" : 907 }, "optimeDate" : ISODate("2013-01-17T00:11:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3653, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:27Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3665, "optime" : { "t" : 1358381488000, "i" : 6661 }, "optimeDate" : ISODate("2013-01-17T00:11:28Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:11:30.692 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:11:31.494 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:11:31.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:11:32.692 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3671, "optime" : { "t" : 1358381492000, "i" : 8338 }, "optimeDate" : ISODate("2013-01-17T00:11:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3660, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3660, "optime" : { "t" : 1358381491000, "i" : 1634 }, "optimeDate" : ISODate("2013-01-17T00:11:31Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:32Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3670, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3657, "optime" : { "t" : 1358381488000, "i" : 6661 }, "optimeDate" : ISODate("2013-01-17T00:11:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:31Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3657, "optime" : { "t" : 1358381492000, "i" : 1201 }, "optimeDate" : ISODate("2013-01-17T00:11:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3657, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3669, "optime" : { "t" : 1358381492000, "i" : 8040 }, "optimeDate" : ISODate("2013-01-17T00:11:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:11:33.281 [conn219] end connection 127.0.0.3:53951 (3 connections now open) m31002| Wed Jan 16 19:11:33.281 [initandlisten] connection accepted from 127.0.0.3:50444 #221 (4 connections now open) m31000| Wed Jan 16 19:11:34.692 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:11:36.125 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:11:36.693 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3676, "optime" : { "t" : 1358381497000, "i" : 397 }, "optimeDate" : ISODate("2013-01-17T00:11:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3665, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3665, "optime" : { "t" : 1358381496000, "i" : 8955 }, "optimeDate" : ISODate("2013-01-17T00:11:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3675, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3662, "optime" : { "t" : 1358381492000, "i" : 8338 }, "optimeDate" : ISODate("2013-01-17T00:11:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:36Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3662, "optime" : { "t" : 1358381496000, "i" : 1030 }, "optimeDate" : ISODate("2013-01-17T00:11:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3662, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3674, "optime" : { "t" : 1358381497000, "i" : 137 }, "optimeDate" : ISODate("2013-01-17T00:11:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:11:38.693 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:11:40.693 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3680, "optime" : { "t" : 1358381501000, "i" : 1544 }, "optimeDate" : ISODate("2013-01-17T00:11:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3669, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3669, "optime" : { "t" : 1358381497000, "i" : 397 }, "optimeDate" : ISODate("2013-01-17T00:11:37Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:40Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3679, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3666, "optime" : { "t" : 1358381497000, "i" : 397 }, "optimeDate" : ISODate("2013-01-17T00:11:37Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:41Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3666, "optime" : { "t" : 1358381500000, "i" : 80 }, "optimeDate" : ISODate("2013-01-17T00:11:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3666, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:39Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3678, "optime" : { "t" : 1358381501000, "i" : 1544 }, "optimeDate" : ISODate("2013-01-17T00:11:41Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:11:41.495 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:11:41.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:11:42.002 [conn220] end connection 127.0.0.2:38162 (3 connections now open) m31002| Wed Jan 16 19:11:42.502 [initandlisten] connection accepted from 127.0.0.2:36443 #222 (4 connections now open) m31000| Wed Jan 16 19:11:42.694 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:11:44.694 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3684, "optime" : { "t" : 1358381505000, "i" : 3646 }, "optimeDate" : ISODate("2013-01-17T00:11:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3673, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3673, "optime" : { "t" : 1358381504000, "i" : 6299 }, "optimeDate" : ISODate("2013-01-17T00:11:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3683, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3670, "optime" : { "t" : 1358381505000, "i" : 2521 }, "optimeDate" : ISODate("2013-01-17T00:11:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3670, "optime" : { "t" : 1358381501000, "i" : 1544 }, "optimeDate" : ISODate("2013-01-17T00:11:41Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3670, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3682, "optime" : { "t" : 1358381505000, "i" : 3646 }, "optimeDate" : ISODate("2013-01-17T00:11:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:11:46.126 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:11:46.496 [conn218] end connection 127.0.0.4:34868 (2 connections now open) m31001| Wed Jan 16 19:11:46.694 [conn219] end connection 127.0.0.2:40980 (1 connection now open) m31001| Wed Jan 16 19:11:46.694 [initandlisten] connection accepted from 127.0.0.2:38469 #220 (2 connections now open) m31000| Wed Jan 16 19:11:46.695 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:11:46.996 [initandlisten] connection accepted from 127.0.0.4:58011 #221 (3 connections now open) m31000| Wed Jan 16 19:11:48.695 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:11:48.706 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:323870 161ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3688, "optime" : { "t" : 1358381509000, "i" : 3926 }, "optimeDate" : ISODate("2013-01-17T00:11:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3677, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3677, "optime" : { "t" : 1358381505000, "i" : 3646 }, "optimeDate" : ISODate("2013-01-17T00:11:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:48Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3687, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3674, "optime" : { "t" : 1358381509000, "i" : 2357 }, "optimeDate" : ISODate("2013-01-17T00:11:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3674, "optime" : { "t" : 1358381505000, "i" : 3646 }, "optimeDate" : ISODate("2013-01-17T00:11:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3674, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3686, "optime" : { "t" : 1358381509000, "i" : 3804 }, "optimeDate" : ISODate("2013-01-17T00:11:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:11:50.127 [conn132] end connection 127.0.0.4:35259 (4 connections now open) m31000| Wed Jan 16 19:11:50.127 [initandlisten] connection accepted from 127.0.0.4:49344 #133 (5 connections now open) m31000| Wed Jan 16 19:11:50.695 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:11:51.997 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:11:52.696 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3692, "optime" : { "t" : 1358381513000, "i" : 4997 }, "optimeDate" : ISODate("2013-01-17T00:11:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3681, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3681, "optime" : { "t" : 1358381512000, "i" : 209 }, "optimeDate" : ISODate("2013-01-17T00:11:52Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3691, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3678, "optime" : { "t" : 1358381513000, "i" : 2375 }, "optimeDate" : ISODate("2013-01-17T00:11:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3678, "optime" : { "t" : 1358381509000, "i" : 3926 }, "optimeDate" : ISODate("2013-01-17T00:11:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3678, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:53Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3690, "optime" : { "t" : 1358381513000, "i" : 3882 }, "optimeDate" : ISODate("2013-01-17T00:11:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:11:54.696 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:11:56.129 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:11:56.696 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3696, "optime" : { "t" : 1358381517000, "i" : 6429 }, "optimeDate" : ISODate("2013-01-17T00:11:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3685, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3685, "optime" : { "t" : 1358381517000, "i" : 3844 }, "optimeDate" : ISODate("2013-01-17T00:11:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3695, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3682, "optime" : { "t" : 1358381517000, "i" : 2343 }, "optimeDate" : ISODate("2013-01-17T00:11:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:11:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3682, "optime" : { "t" : 1358381513000, "i" : 4997 }, "optimeDate" : ISODate("2013-01-17T00:11:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:57Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3682, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:11:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3694, "optime" : { "t" : 1358381517000, "i" : 6418 }, "optimeDate" : ISODate("2013-01-17T00:11:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:11:58.697 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:12:00.697 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3700, "optime" : { "t" : 1358381521000, "i" : 8095 }, "optimeDate" : ISODate("2013-01-17T00:12:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3689, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3689, "optime" : { "t" : 1358381517000, "i" : 6429 }, "optimeDate" : ISODate("2013-01-17T00:11:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3699, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3686, "optime" : { "t" : 1358381521000, "i" : 2331 }, "optimeDate" : ISODate("2013-01-17T00:12:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3686, "optime" : { "t" : 1358381517000, "i" : 6429 }, "optimeDate" : ISODate("2013-01-17T00:11:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3686, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3698, "optime" : { "t" : 1358381521000, "i" : 8095 }, "optimeDate" : ISODate("2013-01-17T00:12:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:12:01.998 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:12:02.697 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:12:03.286 [conn221] end connection 127.0.0.3:50444 (3 connections now open) m31002| Wed Jan 16 19:12:03.287 [initandlisten] connection accepted from 127.0.0.3:40358 #223 (4 connections now open) m31001| Wed Jan 16 19:12:03.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:12:04.698 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:12:06.130 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3705, "optime" : { "t" : 1358381526000, "i" : 1163 }, "optimeDate" : ISODate("2013-01-17T00:12:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3694, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3694, "optime" : { "t" : 1358381525000, "i" : 3 }, "optimeDate" : ISODate("2013-01-17T00:12:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:06Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3704, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3691, "optime" : { "t" : 1358381525000, "i" : 2614 }, "optimeDate" : ISODate("2013-01-17T00:12:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3691, "optime" : { "t" : 1358381526000, "i" : 1079 }, "optimeDate" : ISODate("2013-01-17T00:12:06Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3691, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:05Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3703, "optime" : { "t" : 1358381526000, "i" : 1163 }, "optimeDate" : ISODate("2013-01-17T00:12:06Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:12:06.698 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:12:08.699 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3709, "optime" : { "t" : 1358381530000, "i" : 2616 }, "optimeDate" : ISODate("2013-01-17T00:12:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3698, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3698, "optime" : { "t" : 1358381529000, "i" : 7058 }, "optimeDate" : ISODate("2013-01-17T00:12:09Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:10Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3708, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3695, "optime" : { "t" : 1358381529000, "i" : 354 }, "optimeDate" : ISODate("2013-01-17T00:12:09Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:09Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3695, "optime" : { "t" : 1358381530000, "i" : 1095 }, "optimeDate" : ISODate("2013-01-17T00:12:10Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3695, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3707, "optime" : { "t" : 1358381530000, "i" : 2257 }, "optimeDate" : ISODate("2013-01-17T00:12:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:12:10.699 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:12:12.000 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:12:12.700 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:12:13.613 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:211400 105ms m31001| Wed Jan 16 19:12:13.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3713, "optime" : { "t" : 1358381534000, "i" : 3893 }, "optimeDate" : ISODate("2013-01-17T00:12:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3702, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3702, "optime" : { "t" : 1358381530000, "i" : 2616 }, "optimeDate" : ISODate("2013-01-17T00:12:10Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:14Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3712, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3699, "optime" : { "t" : 1358381530000, "i" : 2616 }, "optimeDate" : ISODate("2013-01-17T00:12:10Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3699, "optime" : { "t" : 1358381534000, "i" : 1086 }, "optimeDate" : ISODate("2013-01-17T00:12:14Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3699, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:13Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3711, "optime" : { "t" : 1358381534000, "i" : 3893 }, "optimeDate" : ISODate("2013-01-17T00:12:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:12:14.700 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:12:16.132 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:12:16.700 [conn220] end connection 127.0.0.2:38469 (2 connections now open) m31001| Wed Jan 16 19:12:16.700 [initandlisten] connection accepted from 127.0.0.2:36954 #222 (3 connections now open) m31000| Wed Jan 16 19:12:16.701 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3717, "optime" : { "t" : 1358381538000, "i" : 5028 }, "optimeDate" : ISODate("2013-01-17T00:12:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3706, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3706, "optime" : { "t" : 1358381537000, "i" : 177 }, "optimeDate" : ISODate("2013-01-17T00:12:17Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:18Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3716, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3703, "optime" : { "t" : 1358381534000, "i" : 3893 }, "optimeDate" : ISODate("2013-01-17T00:12:14Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3703, "optime" : { "t" : 1358381538000, "i" : 1144 }, "optimeDate" : ISODate("2013-01-17T00:12:18Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3703, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3715, "optime" : { "t" : 1358381538000, "i" : 4258 }, "optimeDate" : ISODate("2013-01-17T00:12:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:12:18.701 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:12:20.007 [conn222] end connection 127.0.0.2:36443 (3 connections now open) m31000| Wed Jan 16 19:12:20.132 [conn133] end connection 127.0.0.4:49344 (4 connections now open) m31000| Wed Jan 16 19:12:20.132 [initandlisten] connection accepted from 127.0.0.4:43564 #134 (5 connections now open) m31002| Wed Jan 16 19:12:20.507 [initandlisten] connection accepted from 127.0.0.2:35963 #224 (4 connections now open) m31000| Wed Jan 16 19:12:20.702 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:12:22.001 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:12:22.702 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3721, "optime" : { "t" : 1358381542000, "i" : 6749 }, "optimeDate" : ISODate("2013-01-17T00:12:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3710, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3710, "optime" : { "t" : 1358381538000, "i" : 5028 }, "optimeDate" : ISODate("2013-01-17T00:12:18Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:22Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:11:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3720, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3707, "optime" : { "t" : 1358381538000, "i" : 5028 }, "optimeDate" : ISODate("2013-01-17T00:12:18Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:21Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3707, "optime" : { "t" : 1358381542000, "i" : 1477 }, "optimeDate" : ISODate("2013-01-17T00:12:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3707, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3719, "optime" : { "t" : 1358381542000, "i" : 6749 }, "optimeDate" : ISODate("2013-01-17T00:12:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:12:24.501 [conn221] end connection 127.0.0.4:58011 (2 connections now open) m31000| Wed Jan 16 19:12:24.702 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:12:25.001 [initandlisten] connection accepted from 127.0.0.4:51018 #223 (3 connections now open) m31002| Wed Jan 16 19:12:26.134 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:12:26.703 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3726, "optime" : { "t" : 1358381547000, "i" : 335 }, "optimeDate" : ISODate("2013-01-17T00:12:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3715, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3715, "optime" : { "t" : 1358381542000, "i" : 6749 }, "optimeDate" : ISODate("2013-01-17T00:12:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:26Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3725, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3712, "optime" : { "t" : 1358381542000, "i" : 6749 }, "optimeDate" : ISODate("2013-01-17T00:12:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3712, "optime" : { "t" : 1358381546000, "i" : 1110 }, "optimeDate" : ISODate("2013-01-17T00:12:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3712, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3724, "optime" : { "t" : 1358381547000, "i" : 335 }, "optimeDate" : ISODate("2013-01-17T00:12:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:12:28.703 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:12:30.703 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3730, "optime" : { "t" : 1358381551000, "i" : 2097 }, "optimeDate" : ISODate("2013-01-17T00:12:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3719, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3719, "optime" : { "t" : 1358381550000, "i" : 3870 }, "optimeDate" : ISODate("2013-01-17T00:12:30Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3729, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3716, "optime" : { "t" : 1358381547000, "i" : 335 }, "optimeDate" : ISODate("2013-01-17T00:12:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:30Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3716, "optime" : { "t" : 1358381550000, "i" : 644 }, "optimeDate" : ISODate("2013-01-17T00:12:30Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3716, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3728, "optime" : { "t" : 1358381551000, "i" : 2045 }, "optimeDate" : ISODate("2013-01-17T00:12:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:12:32.135 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:12:32.704 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:12:33.292 [conn223] end connection 127.0.0.3:40358 (3 connections now open) m31002| Wed Jan 16 19:12:33.292 [initandlisten] connection accepted from 127.0.0.3:34825 #225 (4 connections now open) m31000| Wed Jan 16 19:12:34.704 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3734, "optime" : { "t" : 1358381555000, "i" : 3590 }, "optimeDate" : ISODate("2013-01-17T00:12:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3723, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3723, "optime" : { "t" : 1358381551000, "i" : 2097 }, "optimeDate" : ISODate("2013-01-17T00:12:31Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:34Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3733, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3720, "optime" : { "t" : 1358381555000, "i" : 2314 }, "optimeDate" : ISODate("2013-01-17T00:12:35Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3720, "optime" : { "t" : 1358381551000, "i" : 2097 }, "optimeDate" : ISODate("2013-01-17T00:12:31Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3720, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:35Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3732, "optime" : { "t" : 1358381555000, "i" : 3449 }, "optimeDate" : ISODate("2013-01-17T00:12:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:12:35.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:12:36.704 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:12:37.503 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:12:38.705 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3738, "optime" : { "t" : 1358381559000, "i" : 5517 }, "optimeDate" : ISODate("2013-01-17T00:12:39Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3727, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3727, "optime" : { "t" : 1358381555000, "i" : 3590 }, "optimeDate" : ISODate("2013-01-17T00:12:35Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:38Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:39Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3737, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3724, "optime" : { "t" : 1358381559000, "i" : 2551 }, "optimeDate" : ISODate("2013-01-17T00:12:39Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:39Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3724, "optime" : { "t" : 1358381555000, "i" : 3590 }, "optimeDate" : ISODate("2013-01-17T00:12:35Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3724, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:39Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3736, "optime" : { "t" : 1358381559000, "i" : 5052 }, "optimeDate" : ISODate("2013-01-17T00:12:39Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:12:40.705 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:12:42.137 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:12:42.705 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3742, "optime" : { "t" : 1358381563000, "i" : 6646 }, "optimeDate" : ISODate("2013-01-17T00:12:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3731, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3731, "optime" : { "t" : 1358381562000, "i" : 3018 }, "optimeDate" : ISODate("2013-01-17T00:12:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3741, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3728, "optime" : { "t" : 1358381563000, "i" : 2958 }, "optimeDate" : ISODate("2013-01-17T00:12:43Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3728, "optime" : { "t" : 1358381559000, "i" : 5517 }, "optimeDate" : ISODate("2013-01-17T00:12:39Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:43Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3728, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:43Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3740, "optime" : { "t" : 1358381563000, "i" : 5890 }, "optimeDate" : ISODate("2013-01-17T00:12:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:12:44.706 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:12:45.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 19:12:46.706 [conn222] end connection 127.0.0.2:36954 (2 connections now open) m31001| Wed Jan 16 19:12:46.706 [initandlisten] connection accepted from 127.0.0.2:52116 #224 (3 connections now open) m31000| Wed Jan 16 19:12:46.706 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:12:47.505 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3746, "optime" : { "t" : 1358381567000, "i" : 7412 }, "optimeDate" : ISODate("2013-01-17T00:12:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3735, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3735, "optime" : { "t" : 1358381563000, "i" : 6646 }, "optimeDate" : ISODate("2013-01-17T00:12:43Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:46Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3745, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3732, "optime" : { "t" : 1358381567000, "i" : 2089 }, "optimeDate" : ISODate("2013-01-17T00:12:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:47Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3732, "optime" : { "t" : 1358381563000, "i" : 6646 }, "optimeDate" : ISODate("2013-01-17T00:12:43Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3732, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3744, "optime" : { "t" : 1358381567000, "i" : 7412 }, "optimeDate" : ISODate("2013-01-17T00:12:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:12:48.707 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:12:50.138 [conn134] end connection 127.0.0.4:43564 (4 connections now open) m31000| Wed Jan 16 19:12:50.138 [initandlisten] connection accepted from 127.0.0.4:45492 #135 (6 connections now open) m31000| Wed Jan 16 19:12:50.707 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 19:12:51.979 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:259633 129ms { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3751, "optime" : { "t" : 1358381572000, "i" : 483 }, "optimeDate" : ISODate("2013-01-17T00:12:52Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3740, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3740, "optime" : { "t" : 1358381567000, "i" : 7412 }, "optimeDate" : ISODate("2013-01-17T00:12:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3750, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3737, "optime" : { "t" : 1358381571000, "i" : 2655 }, "optimeDate" : ISODate("2013-01-17T00:12:51Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3737, "optime" : { "t" : 1358381567000, "i" : 7412 }, "optimeDate" : ISODate("2013-01-17T00:12:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3737, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3749, "optime" : { "t" : 1358381572000, "i" : 303 }, "optimeDate" : ISODate("2013-01-17T00:12:52Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:12:52.139 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:12:52.707 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:12:54.708 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:12:55.226 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:370474 185ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3755, "optime" : { "t" : 1358381576000, "i" : 525 }, "optimeDate" : ISODate("2013-01-17T00:12:56Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3744, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3744, "optime" : { "t" : 1358381575000, "i" : 4682 }, "optimeDate" : ISODate("2013-01-17T00:12:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3754, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3741, "optime" : { "t" : 1358381575000, "i" : 113 }, "optimeDate" : ISODate("2013-01-17T00:12:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:12:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3741, "optime" : { "t" : 1358381572000, "i" : 483 }, "optimeDate" : ISODate("2013-01-17T00:12:52Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:55Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3741, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3753, "optime" : { "t" : 1358381576000, "i" : 446 }, "optimeDate" : ISODate("2013-01-17T00:12:56Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:12:56.708 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:12:57.506 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 19:12:58.012 [conn224] end connection 127.0.0.2:35963 (3 connections now open) m31002| Wed Jan 16 19:12:58.512 [initandlisten] connection accepted from 127.0.0.2:49590 #226 (4 connections now open) m31000| Wed Jan 16 19:12:58.709 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3759, "optime" : { "t" : 1358381580000, "i" : 1644 }, "optimeDate" : ISODate("2013-01-17T00:13:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3748, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3748, "optime" : { "t" : 1358381576000, "i" : 525 }, "optimeDate" : ISODate("2013-01-17T00:12:56Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:12:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3758, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3745, "optime" : { "t" : 1358381579000, "i" : 1720 }, "optimeDate" : ISODate("2013-01-17T00:12:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3745, "optime" : { "t" : 1358381580000, "i" : 1503 }, "optimeDate" : ISODate("2013-01-17T00:13:00Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3745, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3757, "optime" : { "t" : 1358381580000, "i" : 1644 }, "optimeDate" : ISODate("2013-01-17T00:13:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:13:00.709 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:13:02.140 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:13:02.506 [conn223] end connection 127.0.0.4:51018 (2 connections now open) m31000| Wed Jan 16 19:13:02.710 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:13:03.007 [initandlisten] connection accepted from 127.0.0.4:60837 #225 (3 connections now open) m31002| Wed Jan 16 19:13:03.297 [conn225] end connection 127.0.0.3:34825 (3 connections now open) m31002| Wed Jan 16 19:13:03.297 [initandlisten] connection accepted from 127.0.0.3:59200 #227 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3763, "optime" : { "t" : 1358381584000, "i" : 3189 }, "optimeDate" : ISODate("2013-01-17T00:13:04Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3752, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3752, "optime" : { "t" : 1358381583000, "i" : 2369 }, "optimeDate" : ISODate("2013-01-17T00:13:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:04Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3762, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3749, "optime" : { "t" : 1358381583000, "i" : 669 }, "optimeDate" : ISODate("2013-01-17T00:13:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3749, "optime" : { "t" : 1358381584000, "i" : 1349 }, "optimeDate" : ISODate("2013-01-17T00:13:04Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3749, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:03Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3761, "optime" : { "t" : 1358381584000, "i" : 2716 }, "optimeDate" : ISODate("2013-01-17T00:13:04Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:13:04.710 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:13:06.710 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:13:07.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:13:08.008 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3767, "optime" : { "t" : 1358381588000, "i" : 3935 }, "optimeDate" : ISODate("2013-01-17T00:13:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3756, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3756, "optime" : { "t" : 1358381584000, "i" : 3189 }, "optimeDate" : ISODate("2013-01-17T00:13:04Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:08Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3766, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3753, "optime" : { "t" : 1358381584000, "i" : 3189 }, "optimeDate" : ISODate("2013-01-17T00:13:04Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:08Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3753, "optime" : { "t" : 1358381588000, "i" : 1345 }, "optimeDate" : ISODate("2013-01-17T00:13:08Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3753, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3765, "optime" : { "t" : 1358381588000, "i" : 3935 }, "optimeDate" : ISODate("2013-01-17T00:13:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:13:08.711 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:13:10.711 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3771, "optime" : { "t" : 1358381592000, "i" : 5007 }, "optimeDate" : ISODate("2013-01-17T00:13:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3760, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3760, "optime" : { "t" : 1358381588000, "i" : 3935 }, "optimeDate" : ISODate("2013-01-17T00:13:08Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:12Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3770, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3757, "optime" : { "t" : 1358381588000, "i" : 3935 }, "optimeDate" : ISODate("2013-01-17T00:13:08Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3757, "optime" : { "t" : 1358381592000, "i" : 1143 }, "optimeDate" : ISODate("2013-01-17T00:13:12Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3757, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3769, "optime" : { "t" : 1358381592000, "i" : 5007 }, "optimeDate" : ISODate("2013-01-17T00:13:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:13:12.712 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:13:13.008 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:13:14.712 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:13:16.712 [conn224] end connection 127.0.0.2:52116 (2 connections now open) m31001| Wed Jan 16 19:13:16.712 [initandlisten] connection accepted from 127.0.0.2:47527 #226 (3 connections now open) m31000| Wed Jan 16 19:13:16.713 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3775, "optime" : { "t" : 1358381596000, "i" : 6951 }, "optimeDate" : ISODate("2013-01-17T00:13:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3764, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3764, "optime" : { "t" : 1358381595000, "i" : 3038 }, "optimeDate" : ISODate("2013-01-17T00:13:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3774, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3761, "optime" : { "t" : 1358381592000, "i" : 5007 }, "optimeDate" : ISODate("2013-01-17T00:13:12Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:15Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3761, "optime" : { "t" : 1358381596000, "i" : 1280 }, "optimeDate" : ISODate("2013-01-17T00:13:16Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3761, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3773, "optime" : { "t" : 1358381596000, "i" : 6728 }, "optimeDate" : ISODate("2013-01-17T00:13:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:13:17.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:13:18.009 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:13:18.713 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:13:20.143 [conn135] end connection 127.0.0.4:45492 (4 connections now open) m31000| Wed Jan 16 19:13:20.143 [initandlisten] connection accepted from 127.0.0.4:34424 #136 (5 connections now open) m31000| Wed Jan 16 19:13:20.713 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3780, "optime" : { "t" : 1358381601000, "i" : 61 }, "optimeDate" : ISODate("2013-01-17T00:13:21Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3769, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3769, "optime" : { "t" : 1358381596000, "i" : 6951 }, "optimeDate" : ISODate("2013-01-17T00:13:16Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3779, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3766, "optime" : { "t" : 1358381596000, "i" : 6951 }, "optimeDate" : ISODate("2013-01-17T00:13:16Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:20Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3766, "optime" : { "t" : 1358381600000, "i" : 1208 }, "optimeDate" : ISODate("2013-01-17T00:13:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:21Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3766, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3778, "optime" : { "t" : 1358381600000, "i" : 8217 }, "optimeDate" : ISODate("2013-01-17T00:13:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:13:22.714 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:13:23.010 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:13:24.714 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3784, "optime" : { "t" : 1358381605000, "i" : 1830 }, "optimeDate" : ISODate("2013-01-17T00:13:25Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3773, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3773, "optime" : { "t" : 1358381601000, "i" : 61 }, "optimeDate" : ISODate("2013-01-17T00:13:21Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:24Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:12:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3783, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3770, "optime" : { "t" : 1358381601000, "i" : 61 }, "optimeDate" : ISODate("2013-01-17T00:13:21Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:25Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3770, "optime" : { "t" : 1358381604000, "i" : 1063 }, "optimeDate" : ISODate("2013-01-17T00:13:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3770, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3782, "optime" : { "t" : 1358381605000, "i" : 1830 }, "optimeDate" : ISODate("2013-01-17T00:13:25Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:13:26.714 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:13:28.010 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:13:28.715 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:13:29.015 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:859123 429ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3788, "optime" : { "t" : 1358381609000, "i" : 7051 }, "optimeDate" : ISODate("2013-01-17T00:13:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3777, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3777, "optime" : { "t" : 1358381608000, "i" : 1921 }, "optimeDate" : ISODate("2013-01-17T00:13:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3787, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3774, "optime" : { "t" : 1358381609000, "i" : 4406 }, "optimeDate" : ISODate("2013-01-17T00:13:29Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3774, "optime" : { "t" : 1358381605000, "i" : 1830 }, "optimeDate" : ISODate("2013-01-17T00:13:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3774, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:29Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3786, "optime" : { "t" : 1358381609000, "i" : 7051 }, "optimeDate" : ISODate("2013-01-17T00:13:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:13:30.715 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:13:32.715 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:13:33.011 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 19:13:33.303 [conn227] end connection 127.0.0.3:59200 (3 connections now open) m31002| Wed Jan 16 19:13:33.303 [initandlisten] connection accepted from 127.0.0.3:51097 #228 (4 connections now open) STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 19:13:33.705 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:357350 178ms { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3792, "optime" : { "t" : 1358381613000, "i" : 5136 }, "optimeDate" : ISODate("2013-01-17T00:13:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3781, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3781, "optime" : { "t" : 1358381613000, "i" : 4471 }, "optimeDate" : ISODate("2013-01-17T00:13:33Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3791, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3778, "optime" : { "t" : 1358381613000, "i" : 2523 }, "optimeDate" : ISODate("2013-01-17T00:13:33Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3778, "optime" : { "t" : 1358381609000, "i" : 7051 }, "optimeDate" : ISODate("2013-01-17T00:13:29Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:33Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3778, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:33Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3790, "optime" : { "t" : 1358381613000, "i" : 4535 }, "optimeDate" : ISODate("2013-01-17T00:13:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:13:34.716 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:13:36.017 [conn226] end connection 127.0.0.2:49590 (3 connections now open) m31002| Wed Jan 16 19:13:36.517 [initandlisten] connection accepted from 127.0.0.2:34946 #229 (4 connections now open) m31000| Wed Jan 16 19:13:36.716 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3796, "optime" : { "t" : 1358381617000, "i" : 8302 }, "optimeDate" : ISODate("2013-01-17T00:13:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3785, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3785, "optime" : { "t" : 1358381613000, "i" : 5136 }, "optimeDate" : ISODate("2013-01-17T00:13:33Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3795, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3782, "optime" : { "t" : 1358381617000, "i" : 2659 }, "optimeDate" : ISODate("2013-01-17T00:13:37Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3782, "optime" : { "t" : 1358381613000, "i" : 5136 }, "optimeDate" : ISODate("2013-01-17T00:13:33Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3782, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3794, "optime" : { "t" : 1358381617000, "i" : 8302 }, "optimeDate" : ISODate("2013-01-17T00:13:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:13:38.012 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:13:38.716 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:13:39.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 19:13:40.512 [conn225] end connection 127.0.0.4:60837 (2 connections now open) m31000| Wed Jan 16 19:13:40.717 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:13:41.012 [initandlisten] connection accepted from 127.0.0.4:46974 #227 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3801, "optime" : { "t" : 1358381622000, "i" : 367 }, "optimeDate" : ISODate("2013-01-17T00:13:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3790, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3790, "optime" : { "t" : 1358381621000, "i" : 4369 }, "optimeDate" : ISODate("2013-01-17T00:13:41Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3800, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3787, "optime" : { "t" : 1358381621000, "i" : 2864 }, "optimeDate" : ISODate("2013-01-17T00:13:41Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3787, "optime" : { "t" : 1358381617000, "i" : 8302 }, "optimeDate" : ISODate("2013-01-17T00:13:37Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:41Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3787, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:41Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3799, "optime" : { "t" : 1358381622000, "i" : 367 }, "optimeDate" : ISODate("2013-01-17T00:13:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:13:42.717 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:13:43.512 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:13:44.718 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3805, "optime" : { "t" : 1358381626000, "i" : 2346 }, "optimeDate" : ISODate("2013-01-17T00:13:46Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3794, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3794, "optime" : { "t" : 1358381622000, "i" : 367 }, "optimeDate" : ISODate("2013-01-17T00:13:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:46Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3804, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3791, "optime" : { "t" : 1358381625000, "i" : 1687 }, "optimeDate" : ISODate("2013-01-17T00:13:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:46Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3791, "optime" : { "t" : 1358381626000, "i" : 974 }, "optimeDate" : ISODate("2013-01-17T00:13:46Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3791, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3803, "optime" : { "t" : 1358381626000, "i" : 2249 }, "optimeDate" : ISODate("2013-01-17T00:13:46Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:13:46.718 [conn226] end connection 127.0.0.2:47527 (2 connections now open) m31001| Wed Jan 16 19:13:46.718 [initandlisten] connection accepted from 127.0.0.2:54193 #228 (3 connections now open) m31000| Wed Jan 16 19:13:46.718 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:13:48.148 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:13:48.719 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:13:49.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:13:50.148 [conn136] end connection 127.0.0.4:34424 (4 connections now open) m31000| Wed Jan 16 19:13:50.148 [initandlisten] connection accepted from 127.0.0.4:46591 #137 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3809, "optime" : { "t" : 1358381630000, "i" : 3514 }, "optimeDate" : ISODate("2013-01-17T00:13:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3798, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3798, "optime" : { "t" : 1358381626000, "i" : 2346 }, "optimeDate" : ISODate("2013-01-17T00:13:46Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:50Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3808, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3795, "optime" : { "t" : 1358381629000, "i" : 245 }, "optimeDate" : ISODate("2013-01-17T00:13:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3795, "optime" : { "t" : 1358381630000, "i" : 1341 }, "optimeDate" : ISODate("2013-01-17T00:13:50Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3795, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3807, "optime" : { "t" : 1358381630000, "i" : 3514 }, "optimeDate" : ISODate("2013-01-17T00:13:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:13:50.719 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:13:52.720 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:13:53.513 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3813, "optime" : { "t" : 1358381634000, "i" : 5174 }, "optimeDate" : ISODate("2013-01-17T00:13:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3802, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3802, "optime" : { "t" : 1358381634000, "i" : 491 }, "optimeDate" : ISODate("2013-01-17T00:13:54Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:54Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3812, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3799, "optime" : { "t" : 1358381630000, "i" : 3514 }, "optimeDate" : ISODate("2013-01-17T00:13:50Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:53Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3799, "optime" : { "t" : 1358381634000, "i" : 1157 }, "optimeDate" : ISODate("2013-01-17T00:13:54Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3799, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3811, "optime" : { "t" : 1358381634000, "i" : 5174 }, "optimeDate" : ISODate("2013-01-17T00:13:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:13:54.720 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:13:56.720 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:13:58.150 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:13:58.721 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3817, "optime" : { "t" : 1358381638000, "i" : 6969 }, "optimeDate" : ISODate("2013-01-17T00:13:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3806, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3806, "optime" : { "t" : 1358381634000, "i" : 5174 }, "optimeDate" : ISODate("2013-01-17T00:13:54Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:58Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3816, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3803, "optime" : { "t" : 1358381634000, "i" : 5174 }, "optimeDate" : ISODate("2013-01-17T00:13:54Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:13:58Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:13:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3803, "optime" : { "t" : 1358381638000, "i" : 1164 }, "optimeDate" : ISODate("2013-01-17T00:13:58Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3803, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3815, "optime" : { "t" : 1358381638000, "i" : 6598 }, "optimeDate" : ISODate("2013-01-17T00:13:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:14:00.721 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:14:02.721 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3821, "optime" : { "t" : 1358381642000, "i" : 8152 }, "optimeDate" : ISODate("2013-01-17T00:14:02Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3810, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3810, "optime" : { "t" : 1358381638000, "i" : 6969 }, "optimeDate" : ISODate("2013-01-17T00:13:58Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:02Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3820, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3807, "optime" : { "t" : 1358381638000, "i" : 6969 }, "optimeDate" : ISODate("2013-01-17T00:13:58Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3807, "optime" : { "t" : 1358381642000, "i" : 1222 }, "optimeDate" : ISODate("2013-01-17T00:14:02Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3807, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3819, "optime" : { "t" : 1358381642000, "i" : 7909 }, "optimeDate" : ISODate("2013-01-17T00:14:02Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:14:03.308 [conn228] end connection 127.0.0.3:51097 (3 connections now open) m31002| Wed Jan 16 19:14:03.308 [initandlisten] connection accepted from 127.0.0.3:37100 #230 (4 connections now open) m31002| Wed Jan 16 19:14:03.515 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:14:04.722 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:14:06.722 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3826, "optime" : { "t" : 1358381647000, "i" : 948 }, "optimeDate" : ISODate("2013-01-17T00:14:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3815, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3815, "optime" : { "t" : 1358381646000, "i" : 4305 }, "optimeDate" : ISODate("2013-01-17T00:14:06Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3825, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3812, "optime" : { "t" : 1358381642000, "i" : 8152 }, "optimeDate" : ISODate("2013-01-17T00:14:02Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:06Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3812, "optime" : { "t" : 1358381646000, "i" : 1475 }, "optimeDate" : ISODate("2013-01-17T00:14:06Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3812, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3824, "optime" : { "t" : 1358381647000, "i" : 948 }, "optimeDate" : ISODate("2013-01-17T00:14:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:14:08.152 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:14:08.722 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:14:10.723 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:14:11.039 [journal] old journal file will be removed: /data/db/testReplSet-0/journal/j._5 m31000| Wed Jan 16 19:14:11.107 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:345535 172ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3830, "optime" : { "t" : 1358381651000, "i" : 2713 }, "optimeDate" : ISODate("2013-01-17T00:14:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3819, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3819, "optime" : { "t" : 1358381647000, "i" : 948 }, "optimeDate" : ISODate("2013-01-17T00:14:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:10Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3829, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3816, "optime" : { "t" : 1358381647000, "i" : 948 }, "optimeDate" : ISODate("2013-01-17T00:14:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:11Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3816, "optime" : { "t" : 1358381650000, "i" : 205 }, "optimeDate" : ISODate("2013-01-17T00:14:10Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3816, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3828, "optime" : { "t" : 1358381651000, "i" : 1707 }, "optimeDate" : ISODate("2013-01-17T00:14:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:14:11.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:14:12.723 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:14:13.516 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 19:14:14.022 [conn229] end connection 127.0.0.2:34946 (3 connections now open) m31002| Wed Jan 16 19:14:14.522 [initandlisten] connection accepted from 127.0.0.2:44277 #231 (4 connections now open) m31000| Wed Jan 16 19:14:14.723 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3834, "optime" : { "t" : 1358381655000, "i" : 2661 }, "optimeDate" : ISODate("2013-01-17T00:14:15Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3823, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3823, "optime" : { "t" : 1358381654000, "i" : 2973 }, "optimeDate" : ISODate("2013-01-17T00:14:14Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3833, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3820, "optime" : { "t" : 1358381651000, "i" : 2713 }, "optimeDate" : ISODate("2013-01-17T00:14:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:13Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3820, "optime" : { "t" : 1358381651000, "i" : 2713 }, "optimeDate" : ISODate("2013-01-17T00:14:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3820, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3832, "optime" : { "t" : 1358381655000, "i" : 2148 }, "optimeDate" : ISODate("2013-01-17T00:14:15Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:14:16.724 [conn228] end connection 127.0.0.2:54193 (2 connections now open) m31001| Wed Jan 16 19:14:16.724 [initandlisten] connection accepted from 127.0.0.2:49845 #229 (3 connections now open) m31000| Wed Jan 16 19:14:16.724 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:14:18.153 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:14:18.517 [conn227] end connection 127.0.0.4:46974 (2 connections now open) m31000| Wed Jan 16 19:14:18.724 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:14:19.017 [initandlisten] connection accepted from 127.0.0.4:41147 #230 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3838, "optime" : { "t" : 1358381659000, "i" : 4104 }, "optimeDate" : ISODate("2013-01-17T00:14:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3827, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3827, "optime" : { "t" : 1358381655000, "i" : 2661 }, "optimeDate" : ISODate("2013-01-17T00:14:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:18Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3837, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3824, "optime" : { "t" : 1358381659000, "i" : 2584 }, "optimeDate" : ISODate("2013-01-17T00:14:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3824, "optime" : { "t" : 1358381655000, "i" : 2661 }, "optimeDate" : ISODate("2013-01-17T00:14:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3824, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:19Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3836, "optime" : { "t" : 1358381659000, "i" : 3978 }, "optimeDate" : ISODate("2013-01-17T00:14:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:14:20.154 [conn137] end connection 127.0.0.4:46591 (4 connections now open) m31000| Wed Jan 16 19:14:20.154 [initandlisten] connection accepted from 127.0.0.4:48580 #138 (5 connections now open) m31000| Wed Jan 16 19:14:20.725 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:14:21.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:14:22.725 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3842, "optime" : { "t" : 1358381663000, "i" : 5467 }, "optimeDate" : ISODate("2013-01-17T00:14:23Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3831, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3831, "optime" : { "t" : 1358381659000, "i" : 4104 }, "optimeDate" : ISODate("2013-01-17T00:14:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:22Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3841, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3828, "optime" : { "t" : 1358381663000, "i" : 2753 }, "optimeDate" : ISODate("2013-01-17T00:14:23Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3828, "optime" : { "t" : 1358381659000, "i" : 4104 }, "optimeDate" : ISODate("2013-01-17T00:14:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3828, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3840, "optime" : { "t" : 1358381663000, "i" : 5467 }, "optimeDate" : ISODate("2013-01-17T00:14:23Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:14:24.018 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:14:24.725 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:14:26.726 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3846, "optime" : { "t" : 1358381667000, "i" : 7146 }, "optimeDate" : ISODate("2013-01-17T00:14:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3835, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3835, "optime" : { "t" : 1358381667000, "i" : 1 }, "optimeDate" : ISODate("2013-01-17T00:14:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:13:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3845, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3832, "optime" : { "t" : 1358381667000, "i" : 3131 }, "optimeDate" : ISODate("2013-01-17T00:14:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3832, "optime" : { "t" : 1358381663000, "i" : 5467 }, "optimeDate" : ISODate("2013-01-17T00:14:23Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:27Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3832, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:27Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3844, "optime" : { "t" : 1358381667000, "i" : 6754 }, "optimeDate" : ISODate("2013-01-17T00:14:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:14:28.726 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:14:29.019 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:14:30.726 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3850, "optime" : { "t" : 1358381671000, "i" : 7219 }, "optimeDate" : ISODate("2013-01-17T00:14:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3839, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3839, "optime" : { "t" : 1358381667000, "i" : 7146 }, "optimeDate" : ISODate("2013-01-17T00:14:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:30Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3849, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3836, "optime" : { "t" : 1358381671000, "i" : 2770 }, "optimeDate" : ISODate("2013-01-17T00:14:31Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:31Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3836, "optime" : { "t" : 1358381667000, "i" : 7146 }, "optimeDate" : ISODate("2013-01-17T00:14:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3836, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3848, "optime" : { "t" : 1358381671000, "i" : 7219 }, "optimeDate" : ISODate("2013-01-17T00:14:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:14:32.727 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:14:33.314 [conn230] end connection 127.0.0.3:37100 (3 connections now open) m31002| Wed Jan 16 19:14:33.314 [initandlisten] connection accepted from 127.0.0.3:37167 #232 (4 connections now open) m31002| Wed Jan 16 19:14:34.020 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:14:34.727 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3854, "optime" : { "t" : 1358381675000, "i" : 8640 }, "optimeDate" : ISODate("2013-01-17T00:14:35Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3843, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3843, "optime" : { "t" : 1358381671000, "i" : 7219 }, "optimeDate" : ISODate("2013-01-17T00:14:31Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:35Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3853, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3840, "optime" : { "t" : 1358381675000, "i" : 2683 }, "optimeDate" : ISODate("2013-01-17T00:14:35Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:35Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3840, "optime" : { "t" : 1358381671000, "i" : 7219 }, "optimeDate" : ISODate("2013-01-17T00:14:31Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3840, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:35Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3852, "optime" : { "t" : 1358381675000, "i" : 8357 }, "optimeDate" : ISODate("2013-01-17T00:14:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:14:36.728 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:14:38.728 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:14:39.020 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3859, "optime" : { "t" : 1358381680000, "i" : 549 }, "optimeDate" : ISODate("2013-01-17T00:14:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3848, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3848, "optime" : { "t" : 1358381679000, "i" : 4370 }, "optimeDate" : ISODate("2013-01-17T00:14:39Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3858, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3845, "optime" : { "t" : 1358381679000, "i" : 2810 }, "optimeDate" : ISODate("2013-01-17T00:14:39Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3845, "optime" : { "t" : 1358381675000, "i" : 8640 }, "optimeDate" : ISODate("2013-01-17T00:14:35Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:39Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3845, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:39Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3857, "optime" : { "t" : 1358381680000, "i" : 271 }, "optimeDate" : ISODate("2013-01-17T00:14:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:14:40.728 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:14:42.729 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:14:43.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:14:43.714 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:310084 155ms m31002| Wed Jan 16 19:14:44.021 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3863, "optime" : { "t" : 1358381684000, "i" : 1835 }, "optimeDate" : ISODate("2013-01-17T00:14:44Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3852, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3852, "optime" : { "t" : 1358381680000, "i" : 549 }, "optimeDate" : ISODate("2013-01-17T00:14:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:44Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3862, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3849, "optime" : { "t" : 1358381683000, "i" : 2147 }, "optimeDate" : ISODate("2013-01-17T00:14:43Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:44Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3849, "optime" : { "t" : 1358381684000, "i" : 1308 }, "optimeDate" : ISODate("2013-01-17T00:14:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3849, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3861, "optime" : { "t" : 1358381684000, "i" : 1766 }, "optimeDate" : ISODate("2013-01-17T00:14:44Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:14:44.729 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:14:46.729 [conn229] end connection 127.0.0.2:49845 (2 connections now open) m31001| Wed Jan 16 19:14:46.730 [initandlisten] connection accepted from 127.0.0.2:47637 #231 (3 connections now open) m31000| Wed Jan 16 19:14:46.730 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3867, "optime" : { "t" : 1358381688000, "i" : 3075 }, "optimeDate" : ISODate("2013-01-17T00:14:48Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3856, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3856, "optime" : { "t" : 1358381684000, "i" : 1835 }, "optimeDate" : ISODate("2013-01-17T00:14:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:48Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3866, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3853, "optime" : { "t" : 1358381687000, "i" : 764 }, "optimeDate" : ISODate("2013-01-17T00:14:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3853, "optime" : { "t" : 1358381688000, "i" : 1350 }, "optimeDate" : ISODate("2013-01-17T00:14:48Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3853, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3865, "optime" : { "t" : 1358381688000, "i" : 3075 }, "optimeDate" : ISODate("2013-01-17T00:14:48Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:14:48.730 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:14:49.022 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:14:50.159 [conn138] end connection 127.0.0.4:48580 (4 connections now open) m31000| Wed Jan 16 19:14:50.159 [initandlisten] connection accepted from 127.0.0.4:44094 #139 (5 connections now open) m31000| Wed Jan 16 19:14:50.731 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:14:51.597 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:227641 113ms m31000| Wed Jan 16 19:14:51.805 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:207753 103ms m31002| Wed Jan 16 19:14:52.027 [conn231] end connection 127.0.0.2:44277 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3871, "optime" : { "t" : 1358381692000, "i" : 4024 }, "optimeDate" : ISODate("2013-01-17T00:14:52Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3860, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3860, "optime" : { "t" : 1358381688000, "i" : 3075 }, "optimeDate" : ISODate("2013-01-17T00:14:48Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:52Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3870, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3857, "optime" : { "t" : 1358381688000, "i" : 3075 }, "optimeDate" : ISODate("2013-01-17T00:14:48Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:51Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3857, "optime" : { "t" : 1358381692000, "i" : 1425 }, "optimeDate" : ISODate("2013-01-17T00:14:52Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3857, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3869, "optime" : { "t" : 1358381692000, "i" : 3842 }, "optimeDate" : ISODate("2013-01-17T00:14:52Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:14:52.528 [initandlisten] connection accepted from 127.0.0.2:42558 #233 (4 connections now open) m31000| Wed Jan 16 19:14:52.731 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:14:53.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:14:54.022 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:14:54.731 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:14:56.522 [conn230] end connection 127.0.0.4:41147 (2 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3875, "optime" : { "t" : 1358381696000, "i" : 5997 }, "optimeDate" : ISODate("2013-01-17T00:14:56Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3864, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3864, "optime" : { "t" : 1358381692000, "i" : 4024 }, "optimeDate" : ISODate("2013-01-17T00:14:52Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:56Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3874, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3861, "optime" : { "t" : 1358381692000, "i" : 4024 }, "optimeDate" : ISODate("2013-01-17T00:14:52Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:14:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3861, "optime" : { "t" : 1358381696000, "i" : 1270 }, "optimeDate" : ISODate("2013-01-17T00:14:56Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3861, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3873, "optime" : { "t" : 1358381696000, "i" : 5997 }, "optimeDate" : ISODate("2013-01-17T00:14:56Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:14:56.732 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:14:57.023 [initandlisten] connection accepted from 127.0.0.4:44807 #232 (3 connections now open) m31000| Wed Jan 16 19:14:58.732 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:14:59.523 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:15:00.732 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3879, "optime" : { "t" : 1358381700000, "i" : 7914 }, "optimeDate" : ISODate("2013-01-17T00:15:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3868, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3868, "optime" : { "t" : 1358381700000, "i" : 658 }, "optimeDate" : ISODate("2013-01-17T00:15:00Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3878, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3865, "optime" : { "t" : 1358381696000, "i" : 5997 }, "optimeDate" : ISODate("2013-01-17T00:14:56Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:14:59Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3865, "optime" : { "t" : 1358381700000, "i" : 1577 }, "optimeDate" : ISODate("2013-01-17T00:15:00Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3865, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:14:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3877, "optime" : { "t" : 1358381700000, "i" : 7914 }, "optimeDate" : ISODate("2013-01-17T00:15:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:15:02.733 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:15:03.319 [conn232] end connection 127.0.0.3:37167 (3 connections now open) m31002| Wed Jan 16 19:15:03.319 [initandlisten] connection accepted from 127.0.0.3:52493 #234 (4 connections now open) m31002| Wed Jan 16 19:15:04.162 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:15:04.733 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3884, "optime" : { "t" : 1358381705000, "i" : 662 }, "optimeDate" : ISODate("2013-01-17T00:15:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3873, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3873, "optime" : { "t" : 1358381705000, "i" : 397 }, "optimeDate" : ISODate("2013-01-17T00:15:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3883, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3870, "optime" : { "t" : 1358381700000, "i" : 7914 }, "optimeDate" : ISODate("2013-01-17T00:15:00Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:04Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3870, "optime" : { "t" : 1358381704000, "i" : 1463 }, "optimeDate" : ISODate("2013-01-17T00:15:04Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:05Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3870, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3882, "optime" : { "t" : 1358381705000, "i" : 556 }, "optimeDate" : ISODate("2013-01-17T00:15:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:15:06.733 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:15:08.734 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3888, "optime" : { "t" : 1358381709000, "i" : 2374 }, "optimeDate" : ISODate("2013-01-17T00:15:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3877, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3877, "optime" : { "t" : 1358381705000, "i" : 662 }, "optimeDate" : ISODate("2013-01-17T00:15:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:08Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3887, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3874, "optime" : { "t" : 1358381705000, "i" : 662 }, "optimeDate" : ISODate("2013-01-17T00:15:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3874, "optime" : { "t" : 1358381708000, "i" : 644 }, "optimeDate" : ISODate("2013-01-17T00:15:08Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3874, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:07Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3886, "optime" : { "t" : 1358381709000, "i" : 2374 }, "optimeDate" : ISODate("2013-01-17T00:15:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:15:09.525 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:15:10.734 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:15:12.734 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3892, "optime" : { "t" : 1358381713000, "i" : 3492 }, "optimeDate" : ISODate("2013-01-17T00:15:13Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3881, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3881, "optime" : { "t" : 1358381712000, "i" : 2010 }, "optimeDate" : ISODate("2013-01-17T00:15:12Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3891, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3878, "optime" : { "t" : 1358381713000, "i" : 1942 }, "optimeDate" : ISODate("2013-01-17T00:15:13Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3878, "optime" : { "t" : 1358381709000, "i" : 2374 }, "optimeDate" : ISODate("2013-01-17T00:15:09Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3878, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:13Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3890, "optime" : { "t" : 1358381713000, "i" : 3492 }, "optimeDate" : ISODate("2013-01-17T00:15:13Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:15:14.163 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:15:14.735 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:15:15.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 19:15:16.735 [conn231] end connection 127.0.0.2:47637 (2 connections now open) m31001| Wed Jan 16 19:15:16.735 [initandlisten] connection accepted from 127.0.0.2:54358 #233 (3 connections now open) m31000| Wed Jan 16 19:15:16.735 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3896, "optime" : { "t" : 1358381717000, "i" : 4910 }, "optimeDate" : ISODate("2013-01-17T00:15:17Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3885, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3885, "optime" : { "t" : 1358381713000, "i" : 3492 }, "optimeDate" : ISODate("2013-01-17T00:15:13Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3895, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3882, "optime" : { "t" : 1358381717000, "i" : 2753 }, "optimeDate" : ISODate("2013-01-17T00:15:17Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3882, "optime" : { "t" : 1358381713000, "i" : 3492 }, "optimeDate" : ISODate("2013-01-17T00:15:13Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3882, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3894, "optime" : { "t" : 1358381717000, "i" : 4910 }, "optimeDate" : ISODate("2013-01-17T00:15:17Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:15:18.736 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:15:19.526 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:15:20.164 [conn139] end connection 127.0.0.4:44094 (4 connections now open) m31000| Wed Jan 16 19:15:20.165 [initandlisten] connection accepted from 127.0.0.4:49038 #140 (5 connections now open) m31000| Wed Jan 16 19:15:20.736 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3900, "optime" : { "t" : 1358381721000, "i" : 5663 }, "optimeDate" : ISODate("2013-01-17T00:15:21Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3889, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3889, "optime" : { "t" : 1358381717000, "i" : 4910 }, "optimeDate" : ISODate("2013-01-17T00:15:17Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3899, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3886, "optime" : { "t" : 1358381721000, "i" : 2888 }, "optimeDate" : ISODate("2013-01-17T00:15:21Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3886, "optime" : { "t" : 1358381717000, "i" : 4910 }, "optimeDate" : ISODate("2013-01-17T00:15:17Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3886, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:21Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3898, "optime" : { "t" : 1358381721000, "i" : 5305 }, "optimeDate" : ISODate("2013-01-17T00:15:21Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:15:22.736 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:15:24.166 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:15:24.737 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:15:25.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3904, "optime" : { "t" : 1358381725000, "i" : 7143 }, "optimeDate" : ISODate("2013-01-17T00:15:25Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3893, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3893, "optime" : { "t" : 1358381725000, "i" : 8 }, "optimeDate" : ISODate("2013-01-17T00:15:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3903, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3890, "optime" : { "t" : 1358381725000, "i" : 2595 }, "optimeDate" : ISODate("2013-01-17T00:15:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:25Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3890, "optime" : { "t" : 1358381721000, "i" : 5663 }, "optimeDate" : ISODate("2013-01-17T00:15:21Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:25Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3890, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3902, "optime" : { "t" : 1358381725000, "i" : 7143 }, "optimeDate" : ISODate("2013-01-17T00:15:25Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:15:26.737 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:15:28.737 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:15:29.527 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 19:15:29.753 [journal] old journal file will be removed: /data/db/testReplSet-2/journal/j._5 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3909, "optime" : { "t" : 1358381730000, "i" : 638 }, "optimeDate" : ISODate("2013-01-17T00:15:30Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3898, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3898, "optime" : { "t" : 1358381725000, "i" : 7143 }, "optimeDate" : ISODate("2013-01-17T00:15:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:28Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3908, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3895, "optime" : { "t" : 1358381729000, "i" : 2356 }, "optimeDate" : ISODate("2013-01-17T00:15:29Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:29Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3895, "optime" : { "t" : 1358381725000, "i" : 7143 }, "optimeDate" : ISODate("2013-01-17T00:15:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3895, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3907, "optime" : { "t" : 1358381729000, "i" : 8424 }, "optimeDate" : ISODate("2013-01-17T00:15:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:15:30.033 [conn233] end connection 127.0.0.2:42558 (3 connections now open) m31002| Wed Jan 16 19:15:30.533 [initandlisten] connection accepted from 127.0.0.2:34176 #235 (4 connections now open) m31000| Wed Jan 16 19:15:30.738 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:15:32.738 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:15:33.324 [conn234] end connection 127.0.0.3:52493 (3 connections now open) m31002| Wed Jan 16 19:15:33.324 [initandlisten] connection accepted from 127.0.0.3:49342 #236 (4 connections now open) m31002| Wed Jan 16 19:15:34.167 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3913, "optime" : { "t" : 1358381734000, "i" : 1621 }, "optimeDate" : ISODate("2013-01-17T00:15:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3902, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3902, "optime" : { "t" : 1358381733000, "i" : 4 }, "optimeDate" : ISODate("2013-01-17T00:15:33Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:34Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3912, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3899, "optime" : { "t" : 1358381733000, "i" : 2618 }, "optimeDate" : ISODate("2013-01-17T00:15:33Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3899, "optime" : { "t" : 1358381734000, "i" : 1561 }, "optimeDate" : ISODate("2013-01-17T00:15:34Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3899, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:33Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3911, "optime" : { "t" : 1358381734000, "i" : 1621 }, "optimeDate" : ISODate("2013-01-17T00:15:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:15:34.528 [conn232] end connection 127.0.0.4:44807 (2 connections now open) m31000| Wed Jan 16 19:15:34.739 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:15:35.028 [initandlisten] connection accepted from 127.0.0.4:59522 #234 (3 connections now open) m31000| Wed Jan 16 19:15:36.739 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3917, "optime" : { "t" : 1358381738000, "i" : 3389 }, "optimeDate" : ISODate("2013-01-17T00:15:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3906, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3906, "optime" : { "t" : 1358381738000, "i" : 224 }, "optimeDate" : ISODate("2013-01-17T00:15:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:38Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3916, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3903, "optime" : { "t" : 1358381737000, "i" : 352 }, "optimeDate" : ISODate("2013-01-17T00:15:37Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:37Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3903, "optime" : { "t" : 1358381738000, "i" : 1421 }, "optimeDate" : ISODate("2013-01-17T00:15:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3903, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3915, "optime" : { "t" : 1358381738000, "i" : 2832 }, "optimeDate" : ISODate("2013-01-17T00:15:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:15:38.739 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:15:40.029 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:15:40.740 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3921, "optime" : { "t" : 1358381742000, "i" : 4116 }, "optimeDate" : ISODate("2013-01-17T00:15:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3910, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3910, "optime" : { "t" : 1358381738000, "i" : 3389 }, "optimeDate" : ISODate("2013-01-17T00:15:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:42Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3920, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3907, "optime" : { "t" : 1358381738000, "i" : 3389 }, "optimeDate" : ISODate("2013-01-17T00:15:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3907, "optime" : { "t" : 1358381742000, "i" : 1317 }, "optimeDate" : ISODate("2013-01-17T00:15:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3907, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:41Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3919, "optime" : { "t" : 1358381742000, "i" : 4116 }, "optimeDate" : ISODate("2013-01-17T00:15:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:15:42.740 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:15:44.741 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:15:45.030 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3925, "optime" : { "t" : 1358381746000, "i" : 6038 }, "optimeDate" : ISODate("2013-01-17T00:15:46Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3914, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3914, "optime" : { "t" : 1358381745000, "i" : 201 }, "optimeDate" : ISODate("2013-01-17T00:15:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:46Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3924, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3911, "optime" : { "t" : 1358381742000, "i" : 4116 }, "optimeDate" : ISODate("2013-01-17T00:15:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3911, "optime" : { "t" : 1358381746000, "i" : 1408 }, "optimeDate" : ISODate("2013-01-17T00:15:46Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3911, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3923, "optime" : { "t" : 1358381746000, "i" : 6038 }, "optimeDate" : ISODate("2013-01-17T00:15:46Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:15:46.741 [conn233] end connection 127.0.0.2:54358 (2 connections now open) m31001| Wed Jan 16 19:15:46.741 [initandlisten] connection accepted from 127.0.0.2:53175 #235 (3 connections now open) m31000| Wed Jan 16 19:15:46.741 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:15:47.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:15:48.742 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:15:50.030 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:15:50.170 [conn140] end connection 127.0.0.4:49038 (4 connections now open) m31000| Wed Jan 16 19:15:50.170 [initandlisten] connection accepted from 127.0.0.4:60367 #141 (5 connections now open) m31000| Wed Jan 16 19:15:50.742 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3929, "optime" : { "t" : 1358381750000, "i" : 7185 }, "optimeDate" : ISODate("2013-01-17T00:15:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3918, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3918, "optime" : { "t" : 1358381750000, "i" : 3971 }, "optimeDate" : ISODate("2013-01-17T00:15:50Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3928, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3915, "optime" : { "t" : 1358381746000, "i" : 6038 }, "optimeDate" : ISODate("2013-01-17T00:15:46Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:50Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3915, "optime" : { "t" : 1358381750000, "i" : 1433 }, "optimeDate" : ISODate("2013-01-17T00:15:50Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3915, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3927, "optime" : { "t" : 1358381750000, "i" : 6592 }, "optimeDate" : ISODate("2013-01-17T00:15:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:15:52.742 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:15:54.743 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 m31002| Wed Jan 16 19:15:55.031 [rsMgr] replSet info two primaries (transiently) { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3934, "optime" : { "t" : 1358381755000, "i" : 87 }, "optimeDate" : ISODate("2013-01-17T00:15:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3923, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3923, "optime" : { "t" : 1358381750000, "i" : 7185 }, "optimeDate" : ISODate("2013-01-17T00:15:50Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:54Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3933, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3920, "optime" : { "t" : 1358381750000, "i" : 7185 }, "optimeDate" : ISODate("2013-01-17T00:15:50Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:55Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3920, "optime" : { "t" : 1358381754000, "i" : 1210 }, "optimeDate" : ISODate("2013-01-17T00:15:54Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3920, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3932, "optime" : { "t" : 1358381754000, "i" : 8574 }, "optimeDate" : ISODate("2013-01-17T00:15:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:15:56.743 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:15:57.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:15:58.743 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3938, "optime" : { "t" : 1358381759000, "i" : 2119 }, "optimeDate" : ISODate("2013-01-17T00:15:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3927, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3927, "optime" : { "t" : 1358381755000, "i" : 87 }, "optimeDate" : ISODate("2013-01-17T00:15:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:58Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3937, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3924, "optime" : { "t" : 1358381755000, "i" : 87 }, "optimeDate" : ISODate("2013-01-17T00:15:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:15:57Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:15:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3924, "optime" : { "t" : 1358381758000, "i" : 416 }, "optimeDate" : ISODate("2013-01-17T00:15:58Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3924, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3936, "optime" : { "t" : 1358381759000, "i" : 1924 }, "optimeDate" : ISODate("2013-01-17T00:15:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:16:00.032 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:16:00.744 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:16:02.744 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:16:03.329 [conn236] end connection 127.0.0.3:49342 (3 connections now open) m31002| Wed Jan 16 19:16:03.329 [initandlisten] connection accepted from 127.0.0.3:41000 #237 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3942, "optime" : { "t" : 1358381763000, "i" : 3644 }, "optimeDate" : ISODate("2013-01-17T00:16:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3931, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3931, "optime" : { "t" : 1358381763000, "i" : 257 }, "optimeDate" : ISODate("2013-01-17T00:16:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3941, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3928, "optime" : { "t" : 1358381763000, "i" : 3281 }, "optimeDate" : ISODate("2013-01-17T00:16:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3928, "optime" : { "t" : 1358381759000, "i" : 2119 }, "optimeDate" : ISODate("2013-01-17T00:15:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:03Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3928, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:03Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3940, "optime" : { "t" : 1358381763000, "i" : 3392 }, "optimeDate" : ISODate("2013-01-17T00:16:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:16:04.744 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:16:05.032 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:16:06.745 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3946, "optime" : { "t" : 1358381767000, "i" : 4424 }, "optimeDate" : ISODate("2013-01-17T00:16:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3935, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3935, "optime" : { "t" : 1358381763000, "i" : 3644 }, "optimeDate" : ISODate("2013-01-17T00:16:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:06Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:15:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3945, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3932, "optime" : { "t" : 1358381767000, "i" : 2699 }, "optimeDate" : ISODate("2013-01-17T00:16:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3932, "optime" : { "t" : 1358381763000, "i" : 3644 }, "optimeDate" : ISODate("2013-01-17T00:16:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3932, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:07Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3944, "optime" : { "t" : 1358381767000, "i" : 4415 }, "optimeDate" : ISODate("2013-01-17T00:16:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:16:08.038 [conn235] end connection 127.0.0.2:34176 (3 connections now open) m31002| Wed Jan 16 19:16:08.538 [initandlisten] connection accepted from 127.0.0.2:46996 #238 (4 connections now open) m31000| Wed Jan 16 19:16:08.745 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:16:10.033 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:16:10.745 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3950, "optime" : { "t" : 1358381771000, "i" : 5883 }, "optimeDate" : ISODate("2013-01-17T00:16:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3939, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3939, "optime" : { "t" : 1358381771000, "i" : 167 }, "optimeDate" : ISODate("2013-01-17T00:16:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3949, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3936, "optime" : { "t" : 1358381771000, "i" : 2878 }, "optimeDate" : ISODate("2013-01-17T00:16:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3936, "optime" : { "t" : 1358381767000, "i" : 4424 }, "optimeDate" : ISODate("2013-01-17T00:16:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:11Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3936, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3948, "optime" : { "t" : 1358381771000, "i" : 5883 }, "optimeDate" : ISODate("2013-01-17T00:16:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:16:12.533 [conn234] end connection 127.0.0.4:59522 (2 connections now open) m31000| Wed Jan 16 19:16:12.746 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:16:13.033 [initandlisten] connection accepted from 127.0.0.4:36942 #236 (3 connections now open) m31000| Wed Jan 16 19:16:14.746 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:16:15.534 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3954, "optime" : { "t" : 1358381775000, "i" : 6882 }, "optimeDate" : ISODate("2013-01-17T00:16:15Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3943, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3943, "optime" : { "t" : 1358381771000, "i" : 5883 }, "optimeDate" : ISODate("2013-01-17T00:16:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:14Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3953, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3940, "optime" : { "t" : 1358381775000, "i" : 2783 }, "optimeDate" : ISODate("2013-01-17T00:16:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:15Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3940, "optime" : { "t" : 1358381771000, "i" : 5883 }, "optimeDate" : ISODate("2013-01-17T00:16:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3940, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3952, "optime" : { "t" : 1358381775000, "i" : 6882 }, "optimeDate" : ISODate("2013-01-17T00:16:15Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:16:16.746 [conn235] end connection 127.0.0.2:53175 (2 connections now open) m31001| Wed Jan 16 19:16:16.746 [initandlisten] connection accepted from 127.0.0.2:45530 #237 (3 connections now open) m31000| Wed Jan 16 19:16:16.747 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:16:18.747 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:16:19.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3958, "optime" : { "t" : 1358381779000, "i" : 8429 }, "optimeDate" : ISODate("2013-01-17T00:16:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3947, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3947, "optime" : { "t" : 1358381775000, "i" : 6882 }, "optimeDate" : ISODate("2013-01-17T00:16:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3957, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3944, "optime" : { "t" : 1358381779000, "i" : 2954 }, "optimeDate" : ISODate("2013-01-17T00:16:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3944, "optime" : { "t" : 1358381775000, "i" : 6882 }, "optimeDate" : ISODate("2013-01-17T00:16:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3944, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:19Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3956, "optime" : { "t" : 1358381779000, "i" : 8429 }, "optimeDate" : ISODate("2013-01-17T00:16:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:16:20.175 [conn141] end connection 127.0.0.4:60367 (4 connections now open) m31000| Wed Jan 16 19:16:20.175 [initandlisten] connection accepted from 127.0.0.4:55878 #142 (5 connections now open) m31002| Wed Jan 16 19:16:20.176 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:16:20.747 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:16:22.748 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 19:16:24.015 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:743954 371ms { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3963, "optime" : { "t" : 1358381784000, "i" : 4023 }, "optimeDate" : ISODate("2013-01-17T00:16:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3952, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3952, "optime" : { "t" : 1358381783000, "i" : 4149 }, "optimeDate" : ISODate("2013-01-17T00:16:23Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:24Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3962, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3949, "optime" : { "t" : 1358381783000, "i" : 2796 }, "optimeDate" : ISODate("2013-01-17T00:16:23Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3949, "optime" : { "t" : 1358381784000, "i" : 2912 }, "optimeDate" : ISODate("2013-01-17T00:16:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3949, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3961, "optime" : { "t" : 1358381784000, "i" : 4023 }, "optimeDate" : ISODate("2013-01-17T00:16:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:16:24.748 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:16:25.535 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:16:26.749 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3967, "optime" : { "t" : 1358381788000, "i" : 3327 }, "optimeDate" : ISODate("2013-01-17T00:16:28Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3956, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3956, "optime" : { "t" : 1358381784000, "i" : 4023 }, "optimeDate" : ISODate("2013-01-17T00:16:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:28Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3966, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3953, "optime" : { "t" : 1358381787000, "i" : 511 }, "optimeDate" : ISODate("2013-01-17T00:16:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:28Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:28Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3953, "optime" : { "t" : 1358381788000, "i" : 1425 }, "optimeDate" : ISODate("2013-01-17T00:16:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3953, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3965, "optime" : { "t" : 1358381788000, "i" : 3327 }, "optimeDate" : ISODate("2013-01-17T00:16:28Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:16:28.749 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:16:29.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:16:30.177 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:16:30.749 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3971, "optime" : { "t" : 1358381792000, "i" : 4838 }, "optimeDate" : ISODate("2013-01-17T00:16:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3960, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3960, "optime" : { "t" : 1358381788000, "i" : 3327 }, "optimeDate" : ISODate("2013-01-17T00:16:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:32Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3970, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3957, "optime" : { "t" : 1358381788000, "i" : 3327 }, "optimeDate" : ISODate("2013-01-17T00:16:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3957, "optime" : { "t" : 1358381792000, "i" : 1505 }, "optimeDate" : ISODate("2013-01-17T00:16:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3957, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:31Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3969, "optime" : { "t" : 1358381792000, "i" : 4669 }, "optimeDate" : ISODate("2013-01-17T00:16:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:16:32.750 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:16:33.334 [conn237] end connection 127.0.0.3:41000 (3 connections now open) m31002| Wed Jan 16 19:16:33.335 [initandlisten] connection accepted from 127.0.0.3:41737 #239 (4 connections now open) m31000| Wed Jan 16 19:16:34.750 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:16:35.537 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3975, "optime" : { "t" : 1358381796000, "i" : 6626 }, "optimeDate" : ISODate("2013-01-17T00:16:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3964, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3964, "optime" : { "t" : 1358381796000, "i" : 302 }, "optimeDate" : ISODate("2013-01-17T00:16:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:36Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3974, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3961, "optime" : { "t" : 1358381792000, "i" : 4838 }, "optimeDate" : ISODate("2013-01-17T00:16:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:35Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3961, "optime" : { "t" : 1358381796000, "i" : 1497 }, "optimeDate" : ISODate("2013-01-17T00:16:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3961, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3973, "optime" : { "t" : 1358381796000, "i" : 6626 }, "optimeDate" : ISODate("2013-01-17T00:16:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:16:36.750 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:16:38.751 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:16:40.179 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:16:40.751 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3979, "optime" : { "t" : 1358381800000, "i" : 7285 }, "optimeDate" : ISODate("2013-01-17T00:16:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3968, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3968, "optime" : { "t" : 1358381796000, "i" : 6626 }, "optimeDate" : ISODate("2013-01-17T00:16:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:40Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3978, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3965, "optime" : { "t" : 1358381796000, "i" : 6626 }, "optimeDate" : ISODate("2013-01-17T00:16:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:40Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3965, "optime" : { "t" : 1358381800000, "i" : 1507 }, "optimeDate" : ISODate("2013-01-17T00:16:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3965, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3977, "optime" : { "t" : 1358381800000, "i" : 6883 }, "optimeDate" : ISODate("2013-01-17T00:16:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:16:42.752 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:16:44.752 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3983, "optime" : { "t" : 1358381804000, "i" : 8373 }, "optimeDate" : ISODate("2013-01-17T00:16:44Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3972, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3972, "optime" : { "t" : 1358381800000, "i" : 7285 }, "optimeDate" : ISODate("2013-01-17T00:16:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:44Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3982, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3969, "optime" : { "t" : 1358381800000, "i" : 7285 }, "optimeDate" : ISODate("2013-01-17T00:16:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3969, "optime" : { "t" : 1358381804000, "i" : 1499 }, "optimeDate" : ISODate("2013-01-17T00:16:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3969, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:43Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3981, "optime" : { "t" : 1358381804000, "i" : 8373 }, "optimeDate" : ISODate("2013-01-17T00:16:44Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:16:45.538 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 19:16:46.044 [conn238] end connection 127.0.0.2:46996 (3 connections now open) m31002| Wed Jan 16 19:16:46.544 [initandlisten] connection accepted from 127.0.0.2:45834 #240 (4 connections now open) m31001| Wed Jan 16 19:16:46.752 [conn237] end connection 127.0.0.2:45530 (2 connections now open) m31001| Wed Jan 16 19:16:46.752 [initandlisten] connection accepted from 127.0.0.2:44446 #238 (3 connections now open) m31000| Wed Jan 16 19:16:46.753 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:16:48.753 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3988, "optime" : { "t" : 1358381809000, "i" : 1112 }, "optimeDate" : ISODate("2013-01-17T00:16:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3977, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3977, "optime" : { "t" : 1358381809000, "i" : 326 }, "optimeDate" : ISODate("2013-01-17T00:16:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3987, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3974, "optime" : { "t" : 1358381804000, "i" : 8373 }, "optimeDate" : ISODate("2013-01-17T00:16:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:48Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3974, "optime" : { "t" : 1358381808000, "i" : 1413 }, "optimeDate" : ISODate("2013-01-17T00:16:48Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:49Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3974, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3986, "optime" : { "t" : 1358381809000, "i" : 1112 }, "optimeDate" : ISODate("2013-01-17T00:16:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:16:50.180 [conn142] end connection 127.0.0.4:55878 (4 connections now open) m31000| Wed Jan 16 19:16:50.180 [initandlisten] connection accepted from 127.0.0.4:52138 #143 (5 connections now open) m31002| Wed Jan 16 19:16:50.181 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:16:50.538 [conn236] end connection 127.0.0.4:36942 (2 connections now open) m31000| Wed Jan 16 19:16:50.753 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:16:51.039 [initandlisten] connection accepted from 127.0.0.4:50677 #239 (3 connections now open) m31001| Wed Jan 16 19:16:51.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:16:52.753 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 19:16:53.241 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:328106 164ms { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3992, "optime" : { "t" : 1358381813000, "i" : 1700 }, "optimeDate" : ISODate("2013-01-17T00:16:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3981, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3981, "optime" : { "t" : 1358381809000, "i" : 1112 }, "optimeDate" : ISODate("2013-01-17T00:16:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:52Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3991, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3978, "optime" : { "t" : 1358381809000, "i" : 1112 }, "optimeDate" : ISODate("2013-01-17T00:16:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3978, "optime" : { "t" : 1358381812000, "i" : 835 }, "optimeDate" : ISODate("2013-01-17T00:16:52Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3978, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3990, "optime" : { "t" : 1358381813000, "i" : 1700 }, "optimeDate" : ISODate("2013-01-17T00:16:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:16:54.754 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:16:56.040 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:16:56.754 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3996, "optime" : { "t" : 1358381817000, "i" : 3754 }, "optimeDate" : ISODate("2013-01-17T00:16:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3985, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3985, "optime" : { "t" : 1358381816000, "i" : 2070 }, "optimeDate" : ISODate("2013-01-17T00:16:56Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3995, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3982, "optime" : { "t" : 1358381817000, "i" : 3280 }, "optimeDate" : ISODate("2013-01-17T00:16:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:16:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3982, "optime" : { "t" : 1358381813000, "i" : 1700 }, "optimeDate" : ISODate("2013-01-17T00:16:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3982, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:16:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3994, "optime" : { "t" : 1358381817000, "i" : 3557 }, "optimeDate" : ISODate("2013-01-17T00:16:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:16:58.754 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:17:00.755 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:17:01.040 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4000, "optime" : { "t" : 1358381821000, "i" : 5543 }, "optimeDate" : ISODate("2013-01-17T00:17:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3989, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3989, "optime" : { "t" : 1358381821000, "i" : 4451 }, "optimeDate" : ISODate("2013-01-17T00:17:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3999, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3986, "optime" : { "t" : 1358381821000, "i" : 2556 }, "optimeDate" : ISODate("2013-01-17T00:17:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3986, "optime" : { "t" : 1358381817000, "i" : 3754 }, "optimeDate" : ISODate("2013-01-17T00:16:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:01Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3986, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3998, "optime" : { "t" : 1358381821000, "i" : 5543 }, "optimeDate" : ISODate("2013-01-17T00:17:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:17:01.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:17:02.755 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:17:03.340 [conn239] end connection 127.0.0.3:41737 (3 connections now open) m31002| Wed Jan 16 19:17:03.340 [initandlisten] connection accepted from 127.0.0.3:49651 #241 (4 connections now open) m31000| Wed Jan 16 19:17:04.755 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4004, "optime" : { "t" : 1358381825000, "i" : 6956 }, "optimeDate" : ISODate("2013-01-17T00:17:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3993, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3993, "optime" : { "t" : 1358381821000, "i" : 5543 }, "optimeDate" : ISODate("2013-01-17T00:17:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:04Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4003, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3990, "optime" : { "t" : 1358381825000, "i" : 2897 }, "optimeDate" : ISODate("2013-01-17T00:17:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3990, "optime" : { "t" : 1358381821000, "i" : 5543 }, "optimeDate" : ISODate("2013-01-17T00:17:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3990, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:05Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4002, "optime" : { "t" : 1358381825000, "i" : 6956 }, "optimeDate" : ISODate("2013-01-17T00:17:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:17:06.041 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:17:06.756 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:17:08.756 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4008, "optime" : { "t" : 1358381829000, "i" : 8489 }, "optimeDate" : ISODate("2013-01-17T00:17:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3997, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3997, "optime" : { "t" : 1358381829000, "i" : 484 }, "optimeDate" : ISODate("2013-01-17T00:17:09Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:16:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4007, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3994, "optime" : { "t" : 1358381829000, "i" : 2888 }, "optimeDate" : ISODate("2013-01-17T00:17:09Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3994, "optime" : { "t" : 1358381825000, "i" : 6956 }, "optimeDate" : ISODate("2013-01-17T00:17:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:09Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3994, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:09Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4006, "optime" : { "t" : 1358381829000, "i" : 8489 }, "optimeDate" : ISODate("2013-01-17T00:17:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:17:10.757 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:17:11.042 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:17:12.757 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4013, "optime" : { "t" : 1358381834000, "i" : 918 }, "optimeDate" : ISODate("2013-01-17T00:17:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4002, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4002, "optime" : { "t" : 1358381834000, "i" : 431 }, "optimeDate" : ISODate("2013-01-17T00:17:14Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4012, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 3999, "optime" : { "t" : 1358381833000, "i" : 2550 }, "optimeDate" : ISODate("2013-01-17T00:17:13Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:13Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3999, "optime" : { "t" : 1358381829000, "i" : 8489 }, "optimeDate" : ISODate("2013-01-17T00:17:09Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 3999, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4011, "optime" : { "t" : 1358381834000, "i" : 918 }, "optimeDate" : ISODate("2013-01-17T00:17:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:17:14.757 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:17:16.042 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:17:16.758 [conn238] end connection 127.0.0.2:44446 (2 connections now open) m31001| Wed Jan 16 19:17:16.758 [initandlisten] connection accepted from 127.0.0.2:39264 #240 (3 connections now open) m31000| Wed Jan 16 19:17:16.758 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4017, "optime" : { "t" : 1358381838000, "i" : 1833 }, "optimeDate" : ISODate("2013-01-17T00:17:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4006, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4006, "optime" : { "t" : 1358381834000, "i" : 918 }, "optimeDate" : ISODate("2013-01-17T00:17:14Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4016, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4003, "optime" : { "t" : 1358381837000, "i" : 1690 }, "optimeDate" : ISODate("2013-01-17T00:17:17Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4003, "optime" : { "t" : 1358381834000, "i" : 918 }, "optimeDate" : ISODate("2013-01-17T00:17:14Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4003, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4015, "optime" : { "t" : 1358381838000, "i" : 1594 }, "optimeDate" : ISODate("2013-01-17T00:17:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:17:18.758 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:17:20.186 [conn143] end connection 127.0.0.4:52138 (4 connections now open) m31000| Wed Jan 16 19:17:20.186 [initandlisten] connection accepted from 127.0.0.4:41890 #144 (5 connections now open) m31000| Wed Jan 16 19:17:20.759 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:17:21.043 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4021, "optime" : { "t" : 1358381842000, "i" : 2965 }, "optimeDate" : ISODate("2013-01-17T00:17:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4010, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4010, "optime" : { "t" : 1358381841000, "i" : 3017 }, "optimeDate" : ISODate("2013-01-17T00:17:21Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:22Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4020, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4007, "optime" : { "t" : 1358381841000, "i" : 1281 }, "optimeDate" : ISODate("2013-01-17T00:17:21Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4007, "optime" : { "t" : 1358381842000, "i" : 1407 }, "optimeDate" : ISODate("2013-01-17T00:17:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4007, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:21Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4019, "optime" : { "t" : 1358381842000, "i" : 2965 }, "optimeDate" : ISODate("2013-01-17T00:17:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:17:22.759 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:17:23.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:17:24.048 [conn240] end connection 127.0.0.2:45834 (3 connections now open) m31002| Wed Jan 16 19:17:24.549 [initandlisten] connection accepted from 127.0.0.2:54657 #242 (4 connections now open) m31000| Wed Jan 16 19:17:24.760 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:17:26.044 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4025, "optime" : { "t" : 1358381846000, "i" : 4319 }, "optimeDate" : ISODate("2013-01-17T00:17:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4014, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4014, "optime" : { "t" : 1358381842000, "i" : 2965 }, "optimeDate" : ISODate("2013-01-17T00:17:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:26Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4024, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4011, "optime" : { "t" : 1358381842000, "i" : 2965 }, "optimeDate" : ISODate("2013-01-17T00:17:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:26Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4011, "optime" : { "t" : 1358381846000, "i" : 1438 }, "optimeDate" : ISODate("2013-01-17T00:17:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4011, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4023, "optime" : { "t" : 1358381846000, "i" : 4319 }, "optimeDate" : ISODate("2013-01-17T00:17:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:17:26.760 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:17:28.544 [conn239] end connection 127.0.0.4:50677 (2 connections now open) m31000| Wed Jan 16 19:17:28.761 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:17:29.044 [initandlisten] connection accepted from 127.0.0.4:57392 #241 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4029, "optime" : { "t" : 1358381850000, "i" : 6116 }, "optimeDate" : ISODate("2013-01-17T00:17:30Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4018, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4018, "optime" : { "t" : 1358381846000, "i" : 4319 }, "optimeDate" : ISODate("2013-01-17T00:17:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:30Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4028, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4015, "optime" : { "t" : 1358381846000, "i" : 4319 }, "optimeDate" : ISODate("2013-01-17T00:17:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4015, "optime" : { "t" : 1358381850000, "i" : 1604 }, "optimeDate" : ISODate("2013-01-17T00:17:30Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4015, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:29Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4027, "optime" : { "t" : 1358381850000, "i" : 5703 }, "optimeDate" : ISODate("2013-01-17T00:17:30Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:17:30.761 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:17:31.545 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:17:32.761 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:17:33.345 [conn241] end connection 127.0.0.3:49651 (3 connections now open) m31002| Wed Jan 16 19:17:33.346 [initandlisten] connection accepted from 127.0.0.3:49346 #243 (4 connections now open) m31001| Wed Jan 16 19:17:33.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:17:33.962 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:518543 259ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4033, "optime" : { "t" : 1358381854000, "i" : 8463 }, "optimeDate" : ISODate("2013-01-17T00:17:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4022, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4022, "optime" : { "t" : 1358381854000, "i" : 6838 }, "optimeDate" : ISODate("2013-01-17T00:17:34Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4032, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4019, "optime" : { "t" : 1358381850000, "i" : 6116 }, "optimeDate" : ISODate("2013-01-17T00:17:30Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:34Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4019, "optime" : { "t" : 1358381854000, "i" : 3544 }, "optimeDate" : ISODate("2013-01-17T00:17:34Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4019, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4031, "optime" : { "t" : 1358381854000, "i" : 8463 }, "optimeDate" : ISODate("2013-01-17T00:17:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:17:34.762 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:17:36.189 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:17:36.762 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:17:38.736 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:227200 113ms m31000| Wed Jan 16 19:17:38.762 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4037, "optime" : { "t" : 1358381858000, "i" : 7422 }, "optimeDate" : ISODate("2013-01-17T00:17:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4026, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4026, "optime" : { "t" : 1358381854000, "i" : 8463 }, "optimeDate" : ISODate("2013-01-17T00:17:34Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:38Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4036, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4023, "optime" : { "t" : 1358381854000, "i" : 8463 }, "optimeDate" : ISODate("2013-01-17T00:17:34Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4023, "optime" : { "t" : 1358381858000, "i" : 1536 }, "optimeDate" : ISODate("2013-01-17T00:17:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4023, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4035, "optime" : { "t" : 1358381858000, "i" : 7422 }, "optimeDate" : ISODate("2013-01-17T00:17:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:17:40.763 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:17:41.546 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:17:42.763 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4041, "optime" : { "t" : 1358381862000, "i" : 8207 }, "optimeDate" : ISODate("2013-01-17T00:17:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4030, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4030, "optime" : { "t" : 1358381862000, "i" : 362 }, "optimeDate" : ISODate("2013-01-17T00:17:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:42Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4040, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4027, "optime" : { "t" : 1358381858000, "i" : 7422 }, "optimeDate" : ISODate("2013-01-17T00:17:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:41Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4027, "optime" : { "t" : 1358381862000, "i" : 1480 }, "optimeDate" : ISODate("2013-01-17T00:17:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4027, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4039, "optime" : { "t" : 1358381862000, "i" : 8152 }, "optimeDate" : ISODate("2013-01-17T00:17:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:17:44.763 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:17:46.190 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:17:46.764 [conn240] end connection 127.0.0.2:39264 (2 connections now open) m31001| Wed Jan 16 19:17:46.764 [initandlisten] connection accepted from 127.0.0.2:35963 #242 (3 connections now open) m31000| Wed Jan 16 19:17:46.764 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4046, "optime" : { "t" : 1358381867000, "i" : 765 }, "optimeDate" : ISODate("2013-01-17T00:17:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4035, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4035, "optime" : { "t" : 1358381867000, "i" : 314 }, "optimeDate" : ISODate("2013-01-17T00:17:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4045, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4032, "optime" : { "t" : 1358381862000, "i" : 8207 }, "optimeDate" : ISODate("2013-01-17T00:17:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:46Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4032, "optime" : { "t" : 1358381866000, "i" : 1573 }, "optimeDate" : ISODate("2013-01-17T00:17:46Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:47Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4032, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4044, "optime" : { "t" : 1358381867000, "i" : 765 }, "optimeDate" : ISODate("2013-01-17T00:17:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:17:48.764 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:17:50.191 [conn144] end connection 127.0.0.4:41890 (4 connections now open) m31000| Wed Jan 16 19:17:50.191 [initandlisten] connection accepted from 127.0.0.4:52153 #145 (5 connections now open) m31000| Wed Jan 16 19:17:50.764 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4050, "optime" : { "t" : 1358381871000, "i" : 2687 }, "optimeDate" : ISODate("2013-01-17T00:17:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4039, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4039, "optime" : { "t" : 1358381867000, "i" : 765 }, "optimeDate" : ISODate("2013-01-17T00:17:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:50Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4049, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4036, "optime" : { "t" : 1358381867000, "i" : 765 }, "optimeDate" : ISODate("2013-01-17T00:17:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4036, "optime" : { "t" : 1358381870000, "i" : 701 }, "optimeDate" : ISODate("2013-01-17T00:17:50Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4036, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4048, "optime" : { "t" : 1358381871000, "i" : 2586 }, "optimeDate" : ISODate("2013-01-17T00:17:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:17:51.547 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:17:52.765 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:17:54.765 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4054, "optime" : { "t" : 1358381875000, "i" : 4133 }, "optimeDate" : ISODate("2013-01-17T00:17:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4043, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4043, "optime" : { "t" : 1358381874000, "i" : 2140 }, "optimeDate" : ISODate("2013-01-17T00:17:54Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4053, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4040, "optime" : { "t" : 1358381875000, "i" : 3055 }, "optimeDate" : ISODate("2013-01-17T00:17:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4040, "optime" : { "t" : 1358381871000, "i" : 2687 }, "optimeDate" : ISODate("2013-01-17T00:17:51Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4040, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4052, "optime" : { "t" : 1358381875000, "i" : 4133 }, "optimeDate" : ISODate("2013-01-17T00:17:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:17:55.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:17:56.192 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:17:56.765 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:17:58.679 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:231606 115ms m31000| Wed Jan 16 19:17:58.766 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4058, "optime" : { "t" : 1358381879000, "i" : 4756 }, "optimeDate" : ISODate("2013-01-17T00:17:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4047, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4047, "optime" : { "t" : 1358381875000, "i" : 4133 }, "optimeDate" : ISODate("2013-01-17T00:17:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:58Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4057, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4044, "optime" : { "t" : 1358381879000, "i" : 3276 }, "optimeDate" : ISODate("2013-01-17T00:17:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:17:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4044, "optime" : { "t" : 1358381875000, "i" : 4133 }, "optimeDate" : ISODate("2013-01-17T00:17:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4044, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:17:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4056, "optime" : { "t" : 1358381879000, "i" : 3939 }, "optimeDate" : ISODate("2013-01-17T00:17:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:18:00.766 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:18:01.548 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 19:18:02.054 [conn242] end connection 127.0.0.2:54657 (3 connections now open) m31002| Wed Jan 16 19:18:02.554 [initandlisten] connection accepted from 127.0.0.2:49436 #244 (4 connections now open) m31000| Wed Jan 16 19:18:02.766 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:18:03.351 [conn243] end connection 127.0.0.3:49346 (3 connections now open) m31002| Wed Jan 16 19:18:03.351 [initandlisten] connection accepted from 127.0.0.3:50859 #245 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4062, "optime" : { "t" : 1358381883000, "i" : 5490 }, "optimeDate" : ISODate("2013-01-17T00:18:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4051, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4051, "optime" : { "t" : 1358381882000, "i" : 150 }, "optimeDate" : ISODate("2013-01-17T00:18:02Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4061, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4048, "optime" : { "t" : 1358381883000, "i" : 2750 }, "optimeDate" : ISODate("2013-01-17T00:18:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4048, "optime" : { "t" : 1358381879000, "i" : 4756 }, "optimeDate" : ISODate("2013-01-17T00:17:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4048, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:03Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4060, "optime" : { "t" : 1358381883000, "i" : 5374 }, "optimeDate" : ISODate("2013-01-17T00:18:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:18:04.767 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:18:05.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:18:06.194 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:18:06.549 [conn241] end connection 127.0.0.4:57392 (2 connections now open) m31000| Wed Jan 16 19:18:06.767 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:18:07.049 [initandlisten] connection accepted from 127.0.0.4:53023 #243 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4066, "optime" : { "t" : 1358381887000, "i" : 6995 }, "optimeDate" : ISODate("2013-01-17T00:18:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4055, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4055, "optime" : { "t" : 1358381887000, "i" : 5244 }, "optimeDate" : ISODate("2013-01-17T00:18:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4065, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4052, "optime" : { "t" : 1358381887000, "i" : 2798 }, "optimeDate" : ISODate("2013-01-17T00:18:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4052, "optime" : { "t" : 1358381883000, "i" : 5490 }, "optimeDate" : ISODate("2013-01-17T00:18:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:07Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4052, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:07Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4064, "optime" : { "t" : 1358381887000, "i" : 6985 }, "optimeDate" : ISODate("2013-01-17T00:18:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:18:08.768 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:18:10.768 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4070, "optime" : { "t" : 1358381891000, "i" : 8091 }, "optimeDate" : ISODate("2013-01-17T00:18:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4059, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4059, "optime" : { "t" : 1358381887000, "i" : 6995 }, "optimeDate" : ISODate("2013-01-17T00:18:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:10Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:17:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4069, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4056, "optime" : { "t" : 1358381891000, "i" : 2634 }, "optimeDate" : ISODate("2013-01-17T00:18:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4056, "optime" : { "t" : 1358381887000, "i" : 6995 }, "optimeDate" : ISODate("2013-01-17T00:18:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4056, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4068, "optime" : { "t" : 1358381891000, "i" : 8061 }, "optimeDate" : ISODate("2013-01-17T00:18:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:18:12.050 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:18:12.768 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:18:14.769 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4075, "optime" : { "t" : 1358381896000, "i" : 1489 }, "optimeDate" : ISODate("2013-01-17T00:18:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4064, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4064, "optime" : { "t" : 1358381895000, "i" : 127 }, "optimeDate" : ISODate("2013-01-17T00:18:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4074, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4061, "optime" : { "t" : 1358381895000, "i" : 2671 }, "optimeDate" : ISODate("2013-01-17T00:18:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4061, "optime" : { "t" : 1358381891000, "i" : 8091 }, "optimeDate" : ISODate("2013-01-17T00:18:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:15Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4061, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:15Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4073, "optime" : { "t" : 1358381896000, "i" : 1330 }, "optimeDate" : ISODate("2013-01-17T00:18:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:18:16.769 [conn242] end connection 127.0.0.2:35963 (2 connections now open) m31001| Wed Jan 16 19:18:16.769 [initandlisten] connection accepted from 127.0.0.2:49836 #244 (3 connections now open) m31000| Wed Jan 16 19:18:16.769 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:18:17.051 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:18:18.770 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:18:20.197 [conn145] end connection 127.0.0.4:52153 (4 connections now open) m31000| Wed Jan 16 19:18:20.197 [initandlisten] connection accepted from 127.0.0.4:53936 #146 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4079, "optime" : { "t" : 1358381900000, "i" : 2459 }, "optimeDate" : ISODate("2013-01-17T00:18:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4068, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4068, "optime" : { "t" : 1358381900000, "i" : 92 }, "optimeDate" : ISODate("2013-01-17T00:18:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4078, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4065, "optime" : { "t" : 1358381899000, "i" : 1889 }, "optimeDate" : ISODate("2013-01-17T00:18:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:19Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4065, "optime" : { "t" : 1358381900000, "i" : 1484 }, "optimeDate" : ISODate("2013-01-17T00:18:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4065, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4077, "optime" : { "t" : 1358381900000, "i" : 2459 }, "optimeDate" : ISODate("2013-01-17T00:18:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:18:20.770 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:18:22.051 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:18:22.771 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4083, "optime" : { "t" : 1358381904000, "i" : 4283 }, "optimeDate" : ISODate("2013-01-17T00:18:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4072, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4072, "optime" : { "t" : 1358381900000, "i" : 2459 }, "optimeDate" : ISODate("2013-01-17T00:18:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:24Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4082, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4069, "optime" : { "t" : 1358381903000, "i" : 471 }, "optimeDate" : ISODate("2013-01-17T00:18:23Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4069, "optime" : { "t" : 1358381904000, "i" : 1623 }, "optimeDate" : ISODate("2013-01-17T00:18:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4069, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4081, "optime" : { "t" : 1358381904000, "i" : 4184 }, "optimeDate" : ISODate("2013-01-17T00:18:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:18:24.771 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:18:26.771 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:18:27.052 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:18:27.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4087, "optime" : { "t" : 1358381908000, "i" : 5315 }, "optimeDate" : ISODate("2013-01-17T00:18:28Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4076, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4076, "optime" : { "t" : 1358381907000, "i" : 533 }, "optimeDate" : ISODate("2013-01-17T00:18:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:28Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4086, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4073, "optime" : { "t" : 1358381904000, "i" : 4283 }, "optimeDate" : ISODate("2013-01-17T00:18:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:28Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4073, "optime" : { "t" : 1358381908000, "i" : 1398 }, "optimeDate" : ISODate("2013-01-17T00:18:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4073, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:27Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4085, "optime" : { "t" : 1358381908000, "i" : 5315 }, "optimeDate" : ISODate("2013-01-17T00:18:28Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:18:28.772 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:18:30.772 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:18:32.053 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4091, "optime" : { "t" : 1358381912000, "i" : 6045 }, "optimeDate" : ISODate("2013-01-17T00:18:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4080, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4080, "optime" : { "t" : 1358381912000, "i" : 4744 }, "optimeDate" : ISODate("2013-01-17T00:18:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4090, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4077, "optime" : { "t" : 1358381908000, "i" : 5315 }, "optimeDate" : ISODate("2013-01-17T00:18:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:32Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4077, "optime" : { "t" : 1358381912000, "i" : 1413 }, "optimeDate" : ISODate("2013-01-17T00:18:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4077, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4089, "optime" : { "t" : 1358381912000, "i" : 5673 }, "optimeDate" : ISODate("2013-01-17T00:18:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:18:32.773 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:18:33.356 [conn245] end connection 127.0.0.3:50859 (3 connections now open) m31002| Wed Jan 16 19:18:33.356 [initandlisten] connection accepted from 127.0.0.3:55900 #246 (4 connections now open) m31000| Wed Jan 16 19:18:34.773 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:18:36.773 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4095, "optime" : { "t" : 1358381916000, "i" : 7314 }, "optimeDate" : ISODate("2013-01-17T00:18:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4084, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4084, "optime" : { "t" : 1358381912000, "i" : 6045 }, "optimeDate" : ISODate("2013-01-17T00:18:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:36Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4094, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4081, "optime" : { "t" : 1358381912000, "i" : 6045 }, "optimeDate" : ISODate("2013-01-17T00:18:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4081, "optime" : { "t" : 1358381916000, "i" : 1356 }, "optimeDate" : ISODate("2013-01-17T00:18:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4081, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:35Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4093, "optime" : { "t" : 1358381916000, "i" : 7072 }, "optimeDate" : ISODate("2013-01-17T00:18:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:18:37.053 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:18:37.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:18:38.774 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:18:40.059 [conn244] end connection 127.0.0.2:49436 (3 connections now open) m31002| Wed Jan 16 19:18:40.559 [initandlisten] connection accepted from 127.0.0.2:57555 #247 (4 connections now open) m31000| Wed Jan 16 19:18:40.774 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4100, "optime" : { "t" : 1358381921000, "i" : 23 }, "optimeDate" : ISODate("2013-01-17T00:18:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4089, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4089, "optime" : { "t" : 1358381920000, "i" : 4915 }, "optimeDate" : ISODate("2013-01-17T00:18:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4099, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4086, "optime" : { "t" : 1358381916000, "i" : 7314 }, "optimeDate" : ISODate("2013-01-17T00:18:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:39Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:41Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4086, "optime" : { "t" : 1358381920000, "i" : 2026 }, "optimeDate" : ISODate("2013-01-17T00:18:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4086, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4098, "optime" : { "t" : 1358381921000, "i" : 23 }, "optimeDate" : ISODate("2013-01-17T00:18:41Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:18:42.054 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:18:42.774 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:18:44.554 [conn243] end connection 127.0.0.4:53023 (2 connections now open) m31000| Wed Jan 16 19:18:44.775 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:18:45.054 [initandlisten] connection accepted from 127.0.0.4:59967 #245 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4104, "optime" : { "t" : 1358381925000, "i" : 933 }, "optimeDate" : ISODate("2013-01-17T00:18:45Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4093, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4093, "optime" : { "t" : 1358381921000, "i" : 23 }, "optimeDate" : ISODate("2013-01-17T00:18:41Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:44Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:45Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4103, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4090, "optime" : { "t" : 1358381921000, "i" : 23 }, "optimeDate" : ISODate("2013-01-17T00:18:41Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:45Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:45Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4090, "optime" : { "t" : 1358381924000, "i" : 1369 }, "optimeDate" : ISODate("2013-01-17T00:18:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4090, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4102, "optime" : { "t" : 1358381925000, "i" : 628 }, "optimeDate" : ISODate("2013-01-17T00:18:45Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:18:46.775 [conn244] end connection 127.0.0.2:49836 (2 connections now open) m31001| Wed Jan 16 19:18:46.775 [initandlisten] connection accepted from 127.0.0.2:38093 #246 (3 connections now open) m31000| Wed Jan 16 19:18:46.775 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:18:47.555 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:18:48.757 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:244087 122ms m31000| Wed Jan 16 19:18:48.775 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4108, "optime" : { "t" : 1358381929000, "i" : 1917 }, "optimeDate" : ISODate("2013-01-17T00:18:49Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4097, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4097, "optime" : { "t" : 1358381925000, "i" : 933 }, "optimeDate" : ISODate("2013-01-17T00:18:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:48Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:49Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4107, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4094, "optime" : { "t" : 1358381925000, "i" : 933 }, "optimeDate" : ISODate("2013-01-17T00:18:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:47Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:49Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4094, "optime" : { "t" : 1358381928000, "i" : 600 }, "optimeDate" : ISODate("2013-01-17T00:18:48Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4094, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4106, "optime" : { "t" : 1358381929000, "i" : 1917 }, "optimeDate" : ISODate("2013-01-17T00:18:49Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:18:50.201 [conn146] end connection 127.0.0.4:53936 (4 connections now open) m31000| Wed Jan 16 19:18:50.201 [initandlisten] connection accepted from 127.0.0.4:56852 #147 (5 connections now open) m31000| Wed Jan 16 19:18:50.776 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:18:52.202 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:18:52.776 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4112, "optime" : { "t" : 1358381933000, "i" : 3483 }, "optimeDate" : ISODate("2013-01-17T00:18:53Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4101, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4101, "optime" : { "t" : 1358381933000, "i" : 515 }, "optimeDate" : ISODate("2013-01-17T00:18:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:53Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4111, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4098, "optime" : { "t" : 1358381933000, "i" : 3085 }, "optimeDate" : ISODate("2013-01-17T00:18:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:53Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4098, "optime" : { "t" : 1358381929000, "i" : 1917 }, "optimeDate" : ISODate("2013-01-17T00:18:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:53Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4098, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:53Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4110, "optime" : { "t" : 1358381933000, "i" : 3483 }, "optimeDate" : ISODate("2013-01-17T00:18:53Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:18:54.776 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:18:56.776 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4116, "optime" : { "t" : 1358381937000, "i" : 4693 }, "optimeDate" : ISODate("2013-01-17T00:18:57Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4105, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4105, "optime" : { "t" : 1358381933000, "i" : 3483 }, "optimeDate" : ISODate("2013-01-17T00:18:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:56Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:57Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4115, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4102, "optime" : { "t" : 1358381937000, "i" : 3405 }, "optimeDate" : ISODate("2013-01-17T00:18:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:18:57Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4102, "optime" : { "t" : 1358381933000, "i" : 3483 }, "optimeDate" : ISODate("2013-01-17T00:18:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4102, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:18:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4114, "optime" : { "t" : 1358381937000, "i" : 4693 }, "optimeDate" : ISODate("2013-01-17T00:18:57Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:18:57.556 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:18:58.777 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:18:59.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:19:00.777 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4120, "optime" : { "t" : 1358381941000, "i" : 5829 }, "optimeDate" : ISODate("2013-01-17T00:19:01Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4109, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4109, "optime" : { "t" : 1358381940000, "i" : 37 }, "optimeDate" : ISODate("2013-01-17T00:19:00Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:01Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4119, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4106, "optime" : { "t" : 1358381941000, "i" : 3014 }, "optimeDate" : ISODate("2013-01-17T00:19:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:01Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4106, "optime" : { "t" : 1358381937000, "i" : 4693 }, "optimeDate" : ISODate("2013-01-17T00:18:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4106, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:01Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4118, "optime" : { "t" : 1358381941000, "i" : 5829 }, "optimeDate" : ISODate("2013-01-17T00:19:01Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:19:02.204 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:19:02.778 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:19:03.362 [conn246] end connection 127.0.0.3:55900 (3 connections now open) m31002| Wed Jan 16 19:19:03.362 [initandlisten] connection accepted from 127.0.0.3:54914 #248 (4 connections now open) m31000| Wed Jan 16 19:19:04.778 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4124, "optime" : { "t" : 1358381945000, "i" : 6943 }, "optimeDate" : ISODate("2013-01-17T00:19:05Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4113, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4113, "optime" : { "t" : 1358381945000, "i" : 4582 }, "optimeDate" : ISODate("2013-01-17T00:19:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:05Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4123, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4110, "optime" : { "t" : 1358381945000, "i" : 2607 }, "optimeDate" : ISODate("2013-01-17T00:19:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:05Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:05Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4110, "optime" : { "t" : 1358381941000, "i" : 5829 }, "optimeDate" : ISODate("2013-01-17T00:19:01Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:05Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4110, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:05Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4122, "optime" : { "t" : 1358381945000, "i" : 6943 }, "optimeDate" : ISODate("2013-01-17T00:19:05Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:19:06.778 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:19:07.558 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:19:08.779 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:19:09.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4129, "optime" : { "t" : 1358381950000, "i" : 25 }, "optimeDate" : ISODate("2013-01-17T00:19:10Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4118, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4118, "optime" : { "t" : 1358381945000, "i" : 6943 }, "optimeDate" : ISODate("2013-01-17T00:19:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:08Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:10Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4128, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4115, "optime" : { "t" : 1358381949000, "i" : 2908 }, "optimeDate" : ISODate("2013-01-17T00:19:09Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:10Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4115, "optime" : { "t" : 1358381945000, "i" : 6943 }, "optimeDate" : ISODate("2013-01-17T00:19:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4115, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:09Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4127, "optime" : { "t" : 1358381949000, "i" : 8418 }, "optimeDate" : ISODate("2013-01-17T00:19:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:19:10.779 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:19:12.206 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:19:12.779 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4133, "optime" : { "t" : 1358381954000, "i" : 1519 }, "optimeDate" : ISODate("2013-01-17T00:19:14Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4122, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4122, "optime" : { "t" : 1358381953000, "i" : 102 }, "optimeDate" : ISODate("2013-01-17T00:19:13Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:14Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4132, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4119, "optime" : { "t" : 1358381953000, "i" : 3009 }, "optimeDate" : ISODate("2013-01-17T00:19:13Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:14Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4119, "optime" : { "t" : 1358381950000, "i" : 25 }, "optimeDate" : ISODate("2013-01-17T00:19:10Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4119, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:13Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4131, "optime" : { "t" : 1358381954000, "i" : 1519 }, "optimeDate" : ISODate("2013-01-17T00:19:14Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:19:14.780 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:19:16.780 [conn246] end connection 127.0.0.2:38093 (2 connections now open) m31001| Wed Jan 16 19:19:16.780 [initandlisten] connection accepted from 127.0.0.2:60334 #247 (3 connections now open) m31000| Wed Jan 16 19:19:16.781 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:19:17.559 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 19:19:18.064 [conn247] end connection 127.0.0.2:57555 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4137, "optime" : { "t" : 1358381958000, "i" : 2895 }, "optimeDate" : ISODate("2013-01-17T00:19:18Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4126, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4126, "optime" : { "t" : 1358381954000, "i" : 1519 }, "optimeDate" : ISODate("2013-01-17T00:19:14Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:18Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:18Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4136, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4123, "optime" : { "t" : 1358381957000, "i" : 1274 }, "optimeDate" : ISODate("2013-01-17T00:19:17Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:17Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:18Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4123, "optime" : { "t" : 1358381958000, "i" : 1635 }, "optimeDate" : ISODate("2013-01-17T00:19:18Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4123, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4135, "optime" : { "t" : 1358381958000, "i" : 2895 }, "optimeDate" : ISODate("2013-01-17T00:19:18Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:19:18.564 [initandlisten] connection accepted from 127.0.0.2:42545 #249 (4 connections now open) m31000| Wed Jan 16 19:19:18.781 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:19:19.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:19:20.207 [conn147] end connection 127.0.0.4:56852 (4 connections now open) m31000| Wed Jan 16 19:19:20.207 [initandlisten] connection accepted from 127.0.0.4:56891 #148 (5 connections now open) m31000| Wed Jan 16 19:19:20.781 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:19:22.208 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4141, "optime" : { "t" : 1358381962000, "i" : 3601 }, "optimeDate" : ISODate("2013-01-17T00:19:22Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4130, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4130, "optime" : { "t" : 1358381958000, "i" : 2895 }, "optimeDate" : ISODate("2013-01-17T00:19:18Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:22Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:22Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4140, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4127, "optime" : { "t" : 1358381961000, "i" : 420 }, "optimeDate" : ISODate("2013-01-17T00:19:21Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:22Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4127, "optime" : { "t" : 1358381962000, "i" : 2229 }, "optimeDate" : ISODate("2013-01-17T00:19:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4127, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:21Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4139, "optime" : { "t" : 1358381962000, "i" : 3316 }, "optimeDate" : ISODate("2013-01-17T00:19:22Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:19:22.560 [conn245] end connection 127.0.0.4:59967 (2 connections now open) m31000| Wed Jan 16 19:19:22.782 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:19:23.060 [initandlisten] connection accepted from 127.0.0.4:46982 #248 (3 connections now open) m31000| Wed Jan 16 19:19:24.782 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4145, "optime" : { "t" : 1358381966000, "i" : 4703 }, "optimeDate" : ISODate("2013-01-17T00:19:26Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4134, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4134, "optime" : { "t" : 1358381966000, "i" : 657 }, "optimeDate" : ISODate("2013-01-17T00:19:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:26Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:26Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:18:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4144, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4131, "optime" : { "t" : 1358381965000, "i" : 71 }, "optimeDate" : ISODate("2013-01-17T00:19:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:25Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:26Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4131, "optime" : { "t" : 1358381966000, "i" : 1849 }, "optimeDate" : ISODate("2013-01-17T00:19:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4131, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4143, "optime" : { "t" : 1358381966000, "i" : 4703 }, "optimeDate" : ISODate("2013-01-17T00:19:26Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:19:26.783 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:19:28.061 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:19:28.783 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4149, "optime" : { "t" : 1358381970000, "i" : 6372 }, "optimeDate" : ISODate("2013-01-17T00:19:30Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4138, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4138, "optime" : { "t" : 1358381966000, "i" : 4703 }, "optimeDate" : ISODate("2013-01-17T00:19:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:30Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:30Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4148, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4135, "optime" : { "t" : 1358381966000, "i" : 4703 }, "optimeDate" : ISODate("2013-01-17T00:19:26Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:30Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:30Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4135, "optime" : { "t" : 1358381970000, "i" : 1425 }, "optimeDate" : ISODate("2013-01-17T00:19:30Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4135, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4147, "optime" : { "t" : 1358381970000, "i" : 5447 }, "optimeDate" : ISODate("2013-01-17T00:19:30Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:19:30.784 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:19:32.784 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:19:33.061 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 19:19:33.367 [conn248] end connection 127.0.0.3:54914 (3 connections now open) m31002| Wed Jan 16 19:19:33.367 [initandlisten] connection accepted from 127.0.0.3:50910 #250 (4 connections now open) m31000| Wed Jan 16 19:19:34.784 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4153, "optime" : { "t" : 1358381974000, "i" : 7725 }, "optimeDate" : ISODate("2013-01-17T00:19:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4142, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4142, "optime" : { "t" : 1358381970000, "i" : 6372 }, "optimeDate" : ISODate("2013-01-17T00:19:30Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:34Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4152, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4139, "optime" : { "t" : 1358381970000, "i" : 6372 }, "optimeDate" : ISODate("2013-01-17T00:19:30Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4139, "optime" : { "t" : 1358381974000, "i" : 2451 }, "optimeDate" : ISODate("2013-01-17T00:19:34Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4139, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:33Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4151, "optime" : { "t" : 1358381974000, "i" : 7725 }, "optimeDate" : ISODate("2013-01-17T00:19:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:19:36.785 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:19:38.062 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:19:38.785 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4157, "optime" : { "t" : 1358381978000, "i" : 9289 }, "optimeDate" : ISODate("2013-01-17T00:19:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4146, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4146, "optime" : { "t" : 1358381978000, "i" : 5342 }, "optimeDate" : ISODate("2013-01-17T00:19:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4156, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4143, "optime" : { "t" : 1358381974000, "i" : 7725 }, "optimeDate" : ISODate("2013-01-17T00:19:34Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:38Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4143, "optime" : { "t" : 1358381978000, "i" : 2371 }, "optimeDate" : ISODate("2013-01-17T00:19:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4143, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4155, "optime" : { "t" : 1358381978000, "i" : 9289 }, "optimeDate" : ISODate("2013-01-17T00:19:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:19:40.785 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:19:41.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:19:42.786 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:19:43.062 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4162, "optime" : { "t" : 1358381983000, "i" : 900 }, "optimeDate" : ISODate("2013-01-17T00:19:43Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4151, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4151, "optime" : { "t" : 1358381978000, "i" : 9289 }, "optimeDate" : ISODate("2013-01-17T00:19:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:42Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:43Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4161, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4148, "optime" : { "t" : 1358381978000, "i" : 9289 }, "optimeDate" : ISODate("2013-01-17T00:19:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:43Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:43Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4148, "optime" : { "t" : 1358381982000, "i" : 1803 }, "optimeDate" : ISODate("2013-01-17T00:19:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4148, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4160, "optime" : { "t" : 1358381983000, "i" : 596 }, "optimeDate" : ISODate("2013-01-17T00:19:43Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:19:44.786 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:19:46.786 [conn247] end connection 127.0.0.2:60334 (2 connections now open) m31001| Wed Jan 16 19:19:46.786 [initandlisten] connection accepted from 127.0.0.2:53011 #249 (3 connections now open) m31000| Wed Jan 16 19:19:46.787 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4166, "optime" : { "t" : 1358381987000, "i" : 2045 }, "optimeDate" : ISODate("2013-01-17T00:19:47Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4155, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4155, "optime" : { "t" : 1358381983000, "i" : 900 }, "optimeDate" : ISODate("2013-01-17T00:19:43Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:46Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:47Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4165, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4152, "optime" : { "t" : 1358381983000, "i" : 900 }, "optimeDate" : ISODate("2013-01-17T00:19:43Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:45Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:47Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4152, "optime" : { "t" : 1358381986000, "i" : 726 }, "optimeDate" : ISODate("2013-01-17T00:19:46Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4152, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4164, "optime" : { "t" : 1358381987000, "i" : 2045 }, "optimeDate" : ISODate("2013-01-17T00:19:47Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:19:48.063 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:19:48.787 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:19:50.212 [conn148] end connection 127.0.0.4:56891 (4 connections now open) m31000| Wed Jan 16 19:19:50.212 [initandlisten] connection accepted from 127.0.0.4:54071 #149 (5 connections now open) m31000| Wed Jan 16 19:19:50.787 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4170, "optime" : { "t" : 1358381991000, "i" : 3871 }, "optimeDate" : ISODate("2013-01-17T00:19:51Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4159, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4159, "optime" : { "t" : 1358381991000, "i" : 263 }, "optimeDate" : ISODate("2013-01-17T00:19:51Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:51Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4169, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4156, "optime" : { "t" : 1358381991000, "i" : 2736 }, "optimeDate" : ISODate("2013-01-17T00:19:51Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:51Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4156, "optime" : { "t" : 1358381987000, "i" : 2045 }, "optimeDate" : ISODate("2013-01-17T00:19:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:51Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4156, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:51Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4168, "optime" : { "t" : 1358381991000, "i" : 3069 }, "optimeDate" : ISODate("2013-01-17T00:19:51Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:19:51.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:19:52.788 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:19:53.064 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:19:54.788 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4174, "optime" : { "t" : 1358381995000, "i" : 5470 }, "optimeDate" : ISODate("2013-01-17T00:19:55Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4163, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4163, "optime" : { "t" : 1358381991000, "i" : 3871 }, "optimeDate" : ISODate("2013-01-17T00:19:51Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:54Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:55Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4173, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4160, "optime" : { "t" : 1358381995000, "i" : 3435 }, "optimeDate" : ISODate("2013-01-17T00:19:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:55Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:55Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4160, "optime" : { "t" : 1358381991000, "i" : 3871 }, "optimeDate" : ISODate("2013-01-17T00:19:51Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4160, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4172, "optime" : { "t" : 1358381995000, "i" : 5470 }, "optimeDate" : ISODate("2013-01-17T00:19:55Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:19:56.069 [conn249] end connection 127.0.0.2:42545 (3 connections now open) m31002| Wed Jan 16 19:19:56.569 [initandlisten] connection accepted from 127.0.0.2:60704 #251 (4 connections now open) m31000| Wed Jan 16 19:19:56.788 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:19:58.064 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:19:58.789 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4178, "optime" : { "t" : 1358381999000, "i" : 7286 }, "optimeDate" : ISODate("2013-01-17T00:19:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4167, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4167, "optime" : { "t" : 1358381999000, "i" : 746 }, "optimeDate" : ISODate("2013-01-17T00:19:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4177, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4164, "optime" : { "t" : 1358381999000, "i" : 3255 }, "optimeDate" : ISODate("2013-01-17T00:19:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:19:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4164, "optime" : { "t" : 1358381995000, "i" : 5470 }, "optimeDate" : ISODate("2013-01-17T00:19:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:59Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4164, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:19:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:19:59Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4176, "optime" : { "t" : 1358381999000, "i" : 6460 }, "optimeDate" : ISODate("2013-01-17T00:19:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:20:00.564 [conn248] end connection 127.0.0.4:46982 (2 connections now open) m31000| Wed Jan 16 19:20:00.789 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:20:01.064 [initandlisten] connection accepted from 127.0.0.4:36111 #250 (3 connections now open) m31000| Wed Jan 16 19:20:02.789 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:20:03.373 [conn250] end connection 127.0.0.3:50910 (3 connections now open) m31002| Wed Jan 16 19:20:03.373 [initandlisten] connection accepted from 127.0.0.3:52111 #252 (4 connections now open) m31002| Wed Jan 16 19:20:03.565 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4182, "optime" : { "t" : 1358382003000, "i" : 8012 }, "optimeDate" : ISODate("2013-01-17T00:20:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4171, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4171, "optime" : { "t" : 1358381999000, "i" : 7286 }, "optimeDate" : ISODate("2013-01-17T00:19:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:02Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4181, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4168, "optime" : { "t" : 1358382003000, "i" : 2675 }, "optimeDate" : ISODate("2013-01-17T00:20:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:03Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:03Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4168, "optime" : { "t" : 1358381999000, "i" : 7286 }, "optimeDate" : ISODate("2013-01-17T00:19:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4168, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4180, "optime" : { "t" : 1358382003000, "i" : 7500 }, "optimeDate" : ISODate("2013-01-17T00:20:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:20:04.790 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:20:06.790 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4187, "optime" : { "t" : 1358382008000, "i" : 802 }, "optimeDate" : ISODate("2013-01-17T00:20:08Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4176, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4176, "optime" : { "t" : 1358382003000, "i" : 8012 }, "optimeDate" : ISODate("2013-01-17T00:20:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:08Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4186, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4173, "optime" : { "t" : 1358382007000, "i" : 2806 }, "optimeDate" : ISODate("2013-01-17T00:20:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:08Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4173, "optime" : { "t" : 1358382003000, "i" : 8012 }, "optimeDate" : ISODate("2013-01-17T00:20:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4173, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:07Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4185, "optime" : { "t" : 1358382008000, "i" : 787 }, "optimeDate" : ISODate("2013-01-17T00:20:08Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:20:08.216 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:20:08.791 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:20:10.791 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4191, "optime" : { "t" : 1358382012000, "i" : 2097 }, "optimeDate" : ISODate("2013-01-17T00:20:12Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4180, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4180, "optime" : { "t" : 1358382011000, "i" : 3890 }, "optimeDate" : ISODate("2013-01-17T00:20:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:12Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:12Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4190, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4177, "optime" : { "t" : 1358382011000, "i" : 2050 }, "optimeDate" : ISODate("2013-01-17T00:20:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:12Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4177, "optime" : { "t" : 1358382012000, "i" : 1709 }, "optimeDate" : ISODate("2013-01-17T00:20:12Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4177, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4189, "optime" : { "t" : 1358382012000, "i" : 2070 }, "optimeDate" : ISODate("2013-01-17T00:20:12Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:20:12.791 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:20:13.567 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:20:13.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:20:14.792 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4195, "optime" : { "t" : 1358382016000, "i" : 3722 }, "optimeDate" : ISODate("2013-01-17T00:20:16Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4184, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4184, "optime" : { "t" : 1358382012000, "i" : 2097 }, "optimeDate" : ISODate("2013-01-17T00:20:12Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:16Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4194, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4181, "optime" : { "t" : 1358382015000, "i" : 228 }, "optimeDate" : ISODate("2013-01-17T00:20:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:16Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:16Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4181, "optime" : { "t" : 1358382016000, "i" : 2307 }, "optimeDate" : ISODate("2013-01-17T00:20:16Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4181, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4193, "optime" : { "t" : 1358382016000, "i" : 3722 }, "optimeDate" : ISODate("2013-01-17T00:20:16Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:20:16.792 [conn249] end connection 127.0.0.2:53011 (2 connections now open) m31001| Wed Jan 16 19:20:16.792 [initandlisten] connection accepted from 127.0.0.2:41007 #251 (3 connections now open) m31000| Wed Jan 16 19:20:16.793 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:20:18.218 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:20:18.793 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:20:20.218 [conn149] end connection 127.0.0.4:54071 (4 connections now open) m31000| Wed Jan 16 19:20:20.218 [initandlisten] connection accepted from 127.0.0.4:56537 #150 (5 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4199, "optime" : { "t" : 1358382020000, "i" : 5188 }, "optimeDate" : ISODate("2013-01-17T00:20:20Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4188, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4188, "optime" : { "t" : 1358382016000, "i" : 3722 }, "optimeDate" : ISODate("2013-01-17T00:20:16Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:20Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:20Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4198, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4185, "optime" : { "t" : 1358382016000, "i" : 3722 }, "optimeDate" : ISODate("2013-01-17T00:20:16Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:20Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4185, "optime" : { "t" : 1358382020000, "i" : 2263 }, "optimeDate" : ISODate("2013-01-17T00:20:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4185, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:19Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4197, "optime" : { "t" : 1358382020000, "i" : 5188 }, "optimeDate" : ISODate("2013-01-17T00:20:20Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:20:20.793 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:20:22.794 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:20:23.568 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:20:23.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4203, "optime" : { "t" : 1358382024000, "i" : 6298 }, "optimeDate" : ISODate("2013-01-17T00:20:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4192, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4192, "optime" : { "t" : 1358382024000, "i" : 502 }, "optimeDate" : ISODate("2013-01-17T00:20:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:24Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4202, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4189, "optime" : { "t" : 1358382020000, "i" : 5188 }, "optimeDate" : ISODate("2013-01-17T00:20:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:23Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4189, "optime" : { "t" : 1358382024000, "i" : 1871 }, "optimeDate" : ISODate("2013-01-17T00:20:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4189, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4201, "optime" : { "t" : 1358382024000, "i" : 6298 }, "optimeDate" : ISODate("2013-01-17T00:20:24Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:20:24.794 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:20:26.795 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:20:28.219 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:20:28.795 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 19:20:29.046 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:844279 422ms { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4208, "optime" : { "t" : 1358382029000, "i" : 1962 }, "optimeDate" : ISODate("2013-01-17T00:20:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4197, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4197, "optime" : { "t" : 1358382029000, "i" : 247 }, "optimeDate" : ISODate("2013-01-17T00:20:29Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4207, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4194, "optime" : { "t" : 1358382024000, "i" : 6298 }, "optimeDate" : ISODate("2013-01-17T00:20:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:28Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4194, "optime" : { "t" : 1358382028000, "i" : 1747 }, "optimeDate" : ISODate("2013-01-17T00:20:28Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:29Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4194, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4206, "optime" : { "t" : 1358382029000, "i" : 1962 }, "optimeDate" : ISODate("2013-01-17T00:20:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:20:30.795 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:20:32.796 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4212, "optime" : { "t" : 1358382033000, "i" : 2601 }, "optimeDate" : ISODate("2013-01-17T00:20:33Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4201, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4201, "optime" : { "t" : 1358382029000, "i" : 1962 }, "optimeDate" : ISODate("2013-01-17T00:20:29Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:32Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:33Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4211, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4198, "optime" : { "t" : 1358382029000, "i" : 1962 }, "optimeDate" : ISODate("2013-01-17T00:20:29Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:33Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4198, "optime" : { "t" : 1358382032000, "i" : 683 }, "optimeDate" : ISODate("2013-01-17T00:20:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4198, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:31Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4210, "optime" : { "t" : 1358382033000, "i" : 2601 }, "optimeDate" : ISODate("2013-01-17T00:20:33Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:20:33.378 [conn252] end connection 127.0.0.3:52111 (3 connections now open) m31002| Wed Jan 16 19:20:33.378 [initandlisten] connection accepted from 127.0.0.3:57578 #253 (4 connections now open) m31002| Wed Jan 16 19:20:33.569 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 19:20:34.074 [conn251] end connection 127.0.0.2:60704 (3 connections now open) m31002| Wed Jan 16 19:20:34.575 [initandlisten] connection accepted from 127.0.0.2:51635 #254 (4 connections now open) m31000| Wed Jan 16 19:20:34.796 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:20:36.796 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4216, "optime" : { "t" : 1358382037000, "i" : 4451 }, "optimeDate" : ISODate("2013-01-17T00:20:37Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4205, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4205, "optime" : { "t" : 1358382037000, "i" : 62 }, "optimeDate" : ISODate("2013-01-17T00:20:37Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:37Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4215, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4202, "optime" : { "t" : 1358382037000, "i" : 2936 }, "optimeDate" : ISODate("2013-01-17T00:20:37Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:37Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4202, "optime" : { "t" : 1358382033000, "i" : 2601 }, "optimeDate" : ISODate("2013-01-17T00:20:33Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:36Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:37Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4202, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:37Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4214, "optime" : { "t" : 1358382037000, "i" : 4320 }, "optimeDate" : ISODate("2013-01-17T00:20:37Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:20:38.221 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:20:38.570 [conn250] end connection 127.0.0.4:36111 (2 connections now open) m31000| Wed Jan 16 19:20:38.797 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:20:39.070 [initandlisten] connection accepted from 127.0.0.4:45863 #252 (3 connections now open) m31000| Wed Jan 16 19:20:40.797 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4220, "optime" : { "t" : 1358382041000, "i" : 6924 }, "optimeDate" : ISODate("2013-01-17T00:20:41Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4209, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4209, "optime" : { "t" : 1358382037000, "i" : 4451 }, "optimeDate" : ISODate("2013-01-17T00:20:37Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:40Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:41Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:40Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4219, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4206, "optime" : { "t" : 1358382041000, "i" : 3807 }, "optimeDate" : ISODate("2013-01-17T00:20:41Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:41Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:41Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4206, "optime" : { "t" : 1358382037000, "i" : 4451 }, "optimeDate" : ISODate("2013-01-17T00:20:37Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4206, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4218, "optime" : { "t" : 1358382041000, "i" : 6924 }, "optimeDate" : ISODate("2013-01-17T00:20:41Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:20:42.797 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:20:44.071 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:20:44.798 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:20:45.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4225, "optime" : { "t" : 1358382046000, "i" : 7 }, "optimeDate" : ISODate("2013-01-17T00:20:46Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4214, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4214, "optime" : { "t" : 1358382041000, "i" : 6924 }, "optimeDate" : ISODate("2013-01-17T00:20:41Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4224, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4211, "optime" : { "t" : 1358382045000, "i" : 2537 }, "optimeDate" : ISODate("2013-01-17T00:20:45Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4211, "optime" : { "t" : 1358382041000, "i" : 6924 }, "optimeDate" : ISODate("2013-01-17T00:20:41Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4211, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:45Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4223, "optime" : { "t" : 1358382046000, "i" : 7 }, "optimeDate" : ISODate("2013-01-17T00:20:46Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:20:46.798 [conn251] end connection 127.0.0.2:41007 (2 connections now open) m31001| Wed Jan 16 19:20:46.798 [initandlisten] connection accepted from 127.0.0.2:34696 #253 (3 connections now open) m31000| Wed Jan 16 19:20:46.799 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:20:48.799 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:20:49.071 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 19:20:50.223 [conn150] end connection 127.0.0.4:56537 (4 connections now open) m31000| Wed Jan 16 19:20:50.224 [initandlisten] connection accepted from 127.0.0.4:44556 #151 (5 connections now open) { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4229, "optime" : { "t" : 1358382050000, "i" : 1629 }, "optimeDate" : ISODate("2013-01-17T00:20:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4218, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4218, "optime" : { "t" : 1358382049000, "i" : 4599 }, "optimeDate" : ISODate("2013-01-17T00:20:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:50Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4228, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4215, "optime" : { "t" : 1358382049000, "i" : 2179 }, "optimeDate" : ISODate("2013-01-17T00:20:49Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:49Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4215, "optime" : { "t" : 1358382050000, "i" : 1503 }, "optimeDate" : ISODate("2013-01-17T00:20:50Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4215, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:49Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4227, "optime" : { "t" : 1358382050000, "i" : 1527 }, "optimeDate" : ISODate("2013-01-17T00:20:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:20:50.800 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:20:52.800 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:20:53.681 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:215387 107ms m31002| Wed Jan 16 19:20:54.072 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4233, "optime" : { "t" : 1358382054000, "i" : 3607 }, "optimeDate" : ISODate("2013-01-17T00:20:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4222, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4222, "optime" : { "t" : 1358382050000, "i" : 1629 }, "optimeDate" : ISODate("2013-01-17T00:20:50Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:54Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:52Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4232, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4219, "optime" : { "t" : 1358382053000, "i" : 1162 }, "optimeDate" : ISODate("2013-01-17T00:20:53Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:54Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4219, "optime" : { "t" : 1358382054000, "i" : 2467 }, "optimeDate" : ISODate("2013-01-17T00:20:54Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4219, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4231, "optime" : { "t" : 1358382054000, "i" : 2884 }, "optimeDate" : ISODate("2013-01-17T00:20:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:20:54.800 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:20:55.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:20:56.801 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4237, "optime" : { "t" : 1358382058000, "i" : 4536 }, "optimeDate" : ISODate("2013-01-17T00:20:58Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4226, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4226, "optime" : { "t" : 1358382054000, "i" : 3607 }, "optimeDate" : ISODate("2013-01-17T00:20:54Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:58Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:58Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:56Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4236, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4223, "optime" : { "t" : 1358382057000, "i" : 86 }, "optimeDate" : ISODate("2013-01-17T00:20:57Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:20:58Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4223, "optime" : { "t" : 1358382058000, "i" : 1696 }, "optimeDate" : ISODate("2013-01-17T00:20:58Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4223, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:56Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:20:57Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4235, "optime" : { "t" : 1358382058000, "i" : 4536 }, "optimeDate" : ISODate("2013-01-17T00:20:58Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:20:58.801 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:20:59.073 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:21:00.802 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4241, "optime" : { "t" : 1358382062000, "i" : 6122 }, "optimeDate" : ISODate("2013-01-17T00:21:02Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4230, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4230, "optime" : { "t" : 1358382062000, "i" : 232 }, "optimeDate" : ISODate("2013-01-17T00:21:02Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:02Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:02Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:20:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4240, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4227, "optime" : { "t" : 1358382058000, "i" : 4536 }, "optimeDate" : ISODate("2013-01-17T00:20:58Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:01Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:02Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4227, "optime" : { "t" : 1358382062000, "i" : 1618 }, "optimeDate" : ISODate("2013-01-17T00:21:02Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4227, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:01Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4239, "optime" : { "t" : 1358382062000, "i" : 5677 }, "optimeDate" : ISODate("2013-01-17T00:21:02Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:21:02.802 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:21:03.383 [conn253] end connection 127.0.0.3:57578 (3 connections now open) m31002| Wed Jan 16 19:21:03.383 [initandlisten] connection accepted from 127.0.0.3:44283 #255 (4 connections now open) m31002| Wed Jan 16 19:21:04.073 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:21:04.802 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:21:06.803 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4245, "optime" : { "t" : 1358382066000, "i" : 7301 }, "optimeDate" : ISODate("2013-01-17T00:21:06Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4234, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4234, "optime" : { "t" : 1358382062000, "i" : 6122 }, "optimeDate" : ISODate("2013-01-17T00:21:02Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:04Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:06Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:06Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4244, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4231, "optime" : { "t" : 1358382062000, "i" : 6122 }, "optimeDate" : ISODate("2013-01-17T00:21:02Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:06Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:06Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4231, "optime" : { "t" : 1358382066000, "i" : 1432 }, "optimeDate" : ISODate("2013-01-17T00:21:06Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4231, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4243, "optime" : { "t" : 1358382066000, "i" : 7301 }, "optimeDate" : ISODate("2013-01-17T00:21:06Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:21:08.803 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:21:09.074 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:21:10.803 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4250, "optime" : { "t" : 1358382071000, "i" : 40 }, "optimeDate" : ISODate("2013-01-17T00:21:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4239, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4239, "optime" : { "t" : 1358382066000, "i" : 7301 }, "optimeDate" : ISODate("2013-01-17T00:21:06Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:10Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4249, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4236, "optime" : { "t" : 1358382066000, "i" : 7301 }, "optimeDate" : ISODate("2013-01-17T00:21:06Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:09Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4236, "optime" : { "t" : 1358382070000, "i" : 1687 }, "optimeDate" : ISODate("2013-01-17T00:21:10Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4236, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:09Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4248, "optime" : { "t" : 1358382070000, "i" : 8545 }, "optimeDate" : ISODate("2013-01-17T00:21:10Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:21:12.080 [conn254] end connection 127.0.0.2:51635 (3 connections now open) m31002| Wed Jan 16 19:21:12.580 [initandlisten] connection accepted from 127.0.0.2:33045 #256 (4 connections now open) m31000| Wed Jan 16 19:21:12.804 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:21:14.075 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:21:14.804 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4254, "optime" : { "t" : 1358382075000, "i" : 1534 }, "optimeDate" : ISODate("2013-01-17T00:21:15Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4243, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4243, "optime" : { "t" : 1358382074000, "i" : 8460 }, "optimeDate" : ISODate("2013-01-17T00:21:14Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4253, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4240, "optime" : { "t" : 1358382071000, "i" : 40 }, "optimeDate" : ISODate("2013-01-17T00:21:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:14Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4240, "optime" : { "t" : 1358382074000, "i" : 1696 }, "optimeDate" : ISODate("2013-01-17T00:21:14Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:15Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4240, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:13Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4252, "optime" : { "t" : 1358382075000, "i" : 1405 }, "optimeDate" : ISODate("2013-01-17T00:21:15Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:21:16.575 [conn252] end connection 127.0.0.4:45863 (2 connections now open) m31001| Wed Jan 16 19:21:16.804 [conn253] end connection 127.0.0.2:34696 (1 connection now open) m31001| Wed Jan 16 19:21:16.804 [initandlisten] connection accepted from 127.0.0.2:60520 #254 (2 connections now open) m31000| Wed Jan 16 19:21:16.805 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:21:17.075 [initandlisten] connection accepted from 127.0.0.4:57923 #255 (3 connections now open) m31001| Wed Jan 16 19:21:17.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:21:18.805 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4258, "optime" : { "t" : 1358382079000, "i" : 3461 }, "optimeDate" : ISODate("2013-01-17T00:21:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4247, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4247, "optime" : { "t" : 1358382075000, "i" : 1534 }, "optimeDate" : ISODate("2013-01-17T00:21:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:17Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:18Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4257, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4244, "optime" : { "t" : 1358382075000, "i" : 1534 }, "optimeDate" : ISODate("2013-01-17T00:21:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4244, "optime" : { "t" : 1358382075000, "i" : 1534 }, "optimeDate" : ISODate("2013-01-17T00:21:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4244, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:16Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4256, "optime" : { "t" : 1358382079000, "i" : 3461 }, "optimeDate" : ISODate("2013-01-17T00:21:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:21:19.576 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:21:20.229 [conn151] end connection 127.0.0.4:44556 (4 connections now open) m31000| Wed Jan 16 19:21:20.229 [initandlisten] connection accepted from 127.0.0.4:58143 #152 (5 connections now open) m31000| Wed Jan 16 19:21:20.805 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:21:22.806 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4262, "optime" : { "t" : 1358382083000, "i" : 4827 }, "optimeDate" : ISODate("2013-01-17T00:21:23Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4251, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4251, "optime" : { "t" : 1358382082000, "i" : 1327 }, "optimeDate" : ISODate("2013-01-17T00:21:22Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:23Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4261, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4248, "optime" : { "t" : 1358382083000, "i" : 3154 }, "optimeDate" : ISODate("2013-01-17T00:21:23Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:23Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4248, "optime" : { "t" : 1358382079000, "i" : 3461 }, "optimeDate" : ISODate("2013-01-17T00:21:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:22Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4248, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:23Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4260, "optime" : { "t" : 1358382083000, "i" : 3965 }, "optimeDate" : ISODate("2013-01-17T00:21:23Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:21:24.230 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:21:24.806 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:21:26.806 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:21:27.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4266, "optime" : { "t" : 1358382087000, "i" : 5802 }, "optimeDate" : ISODate("2013-01-17T00:21:27Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4255, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4255, "optime" : { "t" : 1358382087000, "i" : 4682 }, "optimeDate" : ISODate("2013-01-17T00:21:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:27Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4265, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4252, "optime" : { "t" : 1358382087000, "i" : 3023 }, "optimeDate" : ISODate("2013-01-17T00:21:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:27Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4252, "optime" : { "t" : 1358382083000, "i" : 4827 }, "optimeDate" : ISODate("2013-01-17T00:21:23Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:27Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4252, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:27Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4264, "optime" : { "t" : 1358382087000, "i" : 5802 }, "optimeDate" : ISODate("2013-01-17T00:21:27Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:21:28.807 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:21:29.577 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:21:30.807 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4270, "optime" : { "t" : 1358382091000, "i" : 7224 }, "optimeDate" : ISODate("2013-01-17T00:21:31Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4259, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4259, "optime" : { "t" : 1358382087000, "i" : 5802 }, "optimeDate" : ISODate("2013-01-17T00:21:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:30Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:31Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4269, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4256, "optime" : { "t" : 1358382091000, "i" : 2677 }, "optimeDate" : ISODate("2013-01-17T00:21:31Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:31Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4256, "optime" : { "t" : 1358382087000, "i" : 5802 }, "optimeDate" : ISODate("2013-01-17T00:21:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4256, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:29Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:31Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4268, "optime" : { "t" : 1358382091000, "i" : 7134 }, "optimeDate" : ISODate("2013-01-17T00:21:31Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:21:32.807 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:21:33.389 [conn255] end connection 127.0.0.3:44283 (3 connections now open) m31002| Wed Jan 16 19:21:33.389 [initandlisten] connection accepted from 127.0.0.3:52906 #257 (4 connections now open) m31002| Wed Jan 16 19:21:34.232 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:21:34.808 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4275, "optime" : { "t" : 1358382096000, "i" : 88 }, "optimeDate" : ISODate("2013-01-17T00:21:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4264, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4264, "optime" : { "t" : 1358382095000, "i" : 219 }, "optimeDate" : ISODate("2013-01-17T00:21:35Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4274, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4261, "optime" : { "t" : 1358382095000, "i" : 3023 }, "optimeDate" : ISODate("2013-01-17T00:21:35Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4261, "optime" : { "t" : 1358382091000, "i" : 7224 }, "optimeDate" : ISODate("2013-01-17T00:21:31Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:35Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4261, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:34Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:35Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4273, "optime" : { "t" : 1358382095000, "i" : 8492 }, "optimeDate" : ISODate("2013-01-17T00:21:35Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:21:36.808 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:21:38.809 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:21:39.579 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4279, "optime" : { "t" : 1358382100000, "i" : 1628 }, "optimeDate" : ISODate("2013-01-17T00:21:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4268, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4268, "optime" : { "t" : 1358382100000, "i" : 211 }, "optimeDate" : ISODate("2013-01-17T00:21:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:40Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4278, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4265, "optime" : { "t" : 1358382099000, "i" : 3132 }, "optimeDate" : ISODate("2013-01-17T00:21:39Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:39Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:39Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4265, "optime" : { "t" : 1358382100000, "i" : 1470 }, "optimeDate" : ISODate("2013-01-17T00:21:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4265, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4277, "optime" : { "t" : 1358382100000, "i" : 1628 }, "optimeDate" : ISODate("2013-01-17T00:21:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:21:40.809 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:21:42.809 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:21:43.954 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:726791 363ms m31002| Wed Jan 16 19:21:44.234 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4283, "optime" : { "t" : 1358382104000, "i" : 5819 }, "optimeDate" : ISODate("2013-01-17T00:21:44Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4272, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4272, "optime" : { "t" : 1358382100000, "i" : 1628 }, "optimeDate" : ISODate("2013-01-17T00:21:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:42Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:44Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4282, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4269, "optime" : { "t" : 1358382103000, "i" : 554 }, "optimeDate" : ISODate("2013-01-17T00:21:43Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4269, "optime" : { "t" : 1358382104000, "i" : 4018 }, "optimeDate" : ISODate("2013-01-17T00:21:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4269, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:41Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:43Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4281, "optime" : { "t" : 1358382104000, "i" : 4019 }, "optimeDate" : ISODate("2013-01-17T00:21:44Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:21:44.810 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:21:46.810 [conn254] end connection 127.0.0.2:60520 (2 connections now open) m31001| Wed Jan 16 19:21:46.810 [initandlisten] connection accepted from 127.0.0.2:41402 #256 (3 connections now open) m31000| Wed Jan 16 19:21:46.811 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:21:48.033 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:228279 114ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4287, "optime" : { "t" : 1358382108000, "i" : 4307 }, "optimeDate" : ISODate("2013-01-17T00:21:48Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4276, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4276, "optime" : { "t" : 1358382107000, "i" : 2283 }, "optimeDate" : ISODate("2013-01-17T00:21:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:47Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:48Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:46Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4286, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4273, "optime" : { "t" : 1358382107000, "i" : 728 }, "optimeDate" : ISODate("2013-01-17T00:21:47Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4273, "optime" : { "t" : 1358382108000, "i" : 2710 }, "optimeDate" : ISODate("2013-01-17T00:21:48Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4273, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4285, "optime" : { "t" : 1358382108000, "i" : 3949 }, "optimeDate" : ISODate("2013-01-17T00:21:48Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:21:48.811 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:21:49.580 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:21:49.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:21:50.085 [conn256] end connection 127.0.0.2:33045 (3 connections now open) m31000| Wed Jan 16 19:21:50.234 [conn152] end connection 127.0.0.4:58143 (4 connections now open) m31000| Wed Jan 16 19:21:50.235 [initandlisten] connection accepted from 127.0.0.4:43277 #153 (5 connections now open) m31002| Wed Jan 16 19:21:50.585 [initandlisten] connection accepted from 127.0.0.2:54115 #258 (4 connections now open) m31000| Wed Jan 16 19:21:50.811 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:21:51.607 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:203468 101ms m31000| Wed Jan 16 19:21:51.832 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:241524 120ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4291, "optime" : { "t" : 1358382112000, "i" : 4319 }, "optimeDate" : ISODate("2013-01-17T00:21:52Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4280, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4280, "optime" : { "t" : 1358382108000, "i" : 4307 }, "optimeDate" : ISODate("2013-01-17T00:21:48Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:50Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:52Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:52Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:50Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4290, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4277, "optime" : { "t" : 1358382108000, "i" : 4307 }, "optimeDate" : ISODate("2013-01-17T00:21:48Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:51Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:52Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:52Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4277, "optime" : { "t" : 1358382112000, "i" : 2221 }, "optimeDate" : ISODate("2013-01-17T00:21:52Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:52Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4277, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:51Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4289, "optime" : { "t" : 1358382112000, "i" : 4319 }, "optimeDate" : ISODate("2013-01-17T00:21:52Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:21:52.812 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:21:54.236 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:21:54.580 [conn255] end connection 127.0.0.4:57923 (2 connections now open) m31000| Wed Jan 16 19:21:54.812 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:21:55.081 [initandlisten] connection accepted from 127.0.0.4:35879 #257 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4295, "optime" : { "t" : 1358382116000, "i" : 5722 }, "optimeDate" : ISODate("2013-01-17T00:21:56Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4284, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4284, "optime" : { "t" : 1358382115000, "i" : 889 }, "optimeDate" : ISODate("2013-01-17T00:21:55Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:55Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:56Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:56Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4294, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4281, "optime" : { "t" : 1358382112000, "i" : 4319 }, "optimeDate" : ISODate("2013-01-17T00:21:52Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:55Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:21:56Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4281, "optime" : { "t" : 1358382116000, "i" : 2352 }, "optimeDate" : ISODate("2013-01-17T00:21:56Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:56Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4281, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:54Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:55Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4293, "optime" : { "t" : 1358382116000, "i" : 5493 }, "optimeDate" : ISODate("2013-01-17T00:21:56Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:21:56.812 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:21:58.813 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:21:59.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:22:00.081 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4299, "optime" : { "t" : 1358382120000, "i" : 6855 }, "optimeDate" : ISODate("2013-01-17T00:22:00Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4288, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4288, "optime" : { "t" : 1358382120000, "i" : 4914 }, "optimeDate" : ISODate("2013-01-17T00:22:00Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:00Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:00Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:21:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4298, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4285, "optime" : { "t" : 1358382116000, "i" : 5722 }, "optimeDate" : ISODate("2013-01-17T00:21:56Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:59Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:00Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4285, "optime" : { "t" : 1358382120000, "i" : 2203 }, "optimeDate" : ISODate("2013-01-17T00:22:00Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4285, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:59Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4297, "optime" : { "t" : 1358382120000, "i" : 6855 }, "optimeDate" : ISODate("2013-01-17T00:22:00Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:22:00.813 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:22:02.813 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:22:03.394 [conn257] end connection 127.0.0.3:52906 (3 connections now open) m31002| Wed Jan 16 19:22:03.394 [initandlisten] connection accepted from 127.0.0.3:40624 #259 (4 connections now open) m31000| Wed Jan 16 19:22:04.814 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4303, "optime" : { "t" : 1358382124000, "i" : 8297 }, "optimeDate" : ISODate("2013-01-17T00:22:04Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4292, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4292, "optime" : { "t" : 1358382120000, "i" : 6855 }, "optimeDate" : ISODate("2013-01-17T00:22:00Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:04Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:04Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:04Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4302, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4289, "optime" : { "t" : 1358382120000, "i" : 6855 }, "optimeDate" : ISODate("2013-01-17T00:22:00Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:03Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:04Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4289, "optime" : { "t" : 1358382124000, "i" : 2061 }, "optimeDate" : ISODate("2013-01-17T00:22:04Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:04Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4289, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:02Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:03Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4301, "optime" : { "t" : 1358382124000, "i" : 8297 }, "optimeDate" : ISODate("2013-01-17T00:22:04Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:22:05.082 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:22:06.814 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:22:08.814 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4308, "optime" : { "t" : 1358382129000, "i" : 809 }, "optimeDate" : ISODate("2013-01-17T00:22:09Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4297, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4297, "optime" : { "t" : 1358382128000, "i" : 58 }, "optimeDate" : ISODate("2013-01-17T00:22:08Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:08Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:09Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:21:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:08Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4307, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4294, "optime" : { "t" : 1358382124000, "i" : 8297 }, "optimeDate" : ISODate("2013-01-17T00:22:04Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:07Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:07Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:09Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4294, "optime" : { "t" : 1358382128000, "i" : 1839 }, "optimeDate" : ISODate("2013-01-17T00:22:08Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:08Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4294, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4306, "optime" : { "t" : 1358382129000, "i" : 727 }, "optimeDate" : ISODate("2013-01-17T00:22:09Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:22:10.083 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:22:10.815 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:22:12.815 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4312, "optime" : { "t" : 1358382133000, "i" : 2584 }, "optimeDate" : ISODate("2013-01-17T00:22:13Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4301, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4301, "optime" : { "t" : 1358382133000, "i" : 933 }, "optimeDate" : ISODate("2013-01-17T00:22:13Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:13Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:12Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4311, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4298, "optime" : { "t" : 1358382129000, "i" : 809 }, "optimeDate" : ISODate("2013-01-17T00:22:09Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:11Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:12Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:13Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4298, "optime" : { "t" : 1358382132000, "i" : 472 }, "optimeDate" : ISODate("2013-01-17T00:22:12Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:12Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:13Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4298, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:12Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4310, "optime" : { "t" : 1358382133000, "i" : 2584 }, "optimeDate" : ISODate("2013-01-17T00:22:13Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:22:14.815 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:22:15.083 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:22:16.816 [conn256] end connection 127.0.0.2:41402 (2 connections now open) m31001| Wed Jan 16 19:22:16.816 [initandlisten] connection accepted from 127.0.0.2:57575 #258 (3 connections now open) m31000| Wed Jan 16 19:22:16.816 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4316, "optime" : { "t" : 1358382137000, "i" : 3967 }, "optimeDate" : ISODate("2013-01-17T00:22:17Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4305, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4305, "optime" : { "t" : 1358382133000, "i" : 2584 }, "optimeDate" : ISODate("2013-01-17T00:22:13Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:15Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:16Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:17Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:16Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4315, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4302, "optime" : { "t" : 1358382137000, "i" : 3771 }, "optimeDate" : ISODate("2013-01-17T00:22:17Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:17Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:17Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4302, "optime" : { "t" : 1358382133000, "i" : 2584 }, "optimeDate" : ISODate("2013-01-17T00:22:13Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:16Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4302, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:14Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:17Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4314, "optime" : { "t" : 1358382137000, "i" : 3967 }, "optimeDate" : ISODate("2013-01-17T00:22:17Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:22:18.816 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:22:20.084 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:22:20.240 [conn153] end connection 127.0.0.4:43277 (4 connections now open) m31000| Wed Jan 16 19:22:20.240 [initandlisten] connection accepted from 127.0.0.4:51041 #154 (5 connections now open) m31000| Wed Jan 16 19:22:20.817 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4320, "optime" : { "t" : 1358382141000, "i" : 5355 }, "optimeDate" : ISODate("2013-01-17T00:22:21Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4309, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4309, "optime" : { "t" : 1358382140000, "i" : 1261 }, "optimeDate" : ISODate("2013-01-17T00:22:20Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:20Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:21Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4319, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4306, "optime" : { "t" : 1358382141000, "i" : 3175 }, "optimeDate" : ISODate("2013-01-17T00:22:21Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:21Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:21Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4306, "optime" : { "t" : 1358382137000, "i" : 3967 }, "optimeDate" : ISODate("2013-01-17T00:22:17Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:20Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:20Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4306, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:19Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:21Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4318, "optime" : { "t" : 1358382141000, "i" : 5355 }, "optimeDate" : ISODate("2013-01-17T00:22:21Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:22:21.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:22:22.817 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:22:24.817 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:22:25.085 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4324, "optime" : { "t" : 1358382145000, "i" : 6680 }, "optimeDate" : ISODate("2013-01-17T00:22:25Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4313, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:24Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4313, "optime" : { "t" : 1358382145000, "i" : 4930 }, "optimeDate" : ISODate("2013-01-17T00:22:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:25Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:24Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4323, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4310, "optime" : { "t" : 1358382145000, "i" : 3315 }, "optimeDate" : ISODate("2013-01-17T00:22:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:25Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:25Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4310, "optime" : { "t" : 1358382141000, "i" : 5355 }, "optimeDate" : ISODate("2013-01-17T00:22:21Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:25Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4310, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:24Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:25Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4322, "optime" : { "t" : 1358382145000, "i" : 6680 }, "optimeDate" : ISODate("2013-01-17T00:22:25Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:22:26.818 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:22:28.090 [conn258] end connection 127.0.0.2:54115 (3 connections now open) m31002| Wed Jan 16 19:22:28.590 [initandlisten] connection accepted from 127.0.0.2:41238 #260 (4 connections now open) m31000| Wed Jan 16 19:22:28.818 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4328, "optime" : { "t" : 1358382149000, "i" : 8465 }, "optimeDate" : ISODate("2013-01-17T00:22:29Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4317, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4317, "optime" : { "t" : 1358382145000, "i" : 6680 }, "optimeDate" : ISODate("2013-01-17T00:22:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:28Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:29Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4327, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4314, "optime" : { "t" : 1358382149000, "i" : 3655 }, "optimeDate" : ISODate("2013-01-17T00:22:29Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:29Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:29Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4314, "optime" : { "t" : 1358382145000, "i" : 6680 }, "optimeDate" : ISODate("2013-01-17T00:22:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:28Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:28Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4314, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:27Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:29Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4326, "optime" : { "t" : 1358382149000, "i" : 8446 }, "optimeDate" : ISODate("2013-01-17T00:22:29Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:22:30.086 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:22:30.818 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:22:31.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31001| Wed Jan 16 19:22:32.586 [conn257] end connection 127.0.0.4:35879 (2 connections now open) m31000| Wed Jan 16 19:22:32.819 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:22:33.086 [initandlisten] connection accepted from 127.0.0.4:47623 #259 (3 connections now open) m31002| Wed Jan 16 19:22:33.399 [conn259] end connection 127.0.0.3:40624 (3 connections now open) m31002| Wed Jan 16 19:22:33.399 [initandlisten] connection accepted from 127.0.0.3:48572 #261 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4333, "optime" : { "t" : 1358382154000, "i" : 757 }, "optimeDate" : ISODate("2013-01-17T00:22:34Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4322, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4322, "optime" : { "t" : 1358382153000, "i" : 4606 }, "optimeDate" : ISODate("2013-01-17T00:22:33Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:34Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:32Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4332, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4319, "optime" : { "t" : 1358382153000, "i" : 3010 }, "optimeDate" : ISODate("2013-01-17T00:22:33Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:33Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:34Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4319, "optime" : { "t" : 1358382149000, "i" : 8465 }, "optimeDate" : ISODate("2013-01-17T00:22:29Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:33Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4319, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:32Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:33Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4331, "optime" : { "t" : 1358382154000, "i" : 757 }, "optimeDate" : ISODate("2013-01-17T00:22:34Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:22:34.819 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:22:35.586 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:22:36.820 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4337, "optime" : { "t" : 1358382158000, "i" : 1924 }, "optimeDate" : ISODate("2013-01-17T00:22:38Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4326, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4326, "optime" : { "t" : 1358382154000, "i" : 757 }, "optimeDate" : ISODate("2013-01-17T00:22:34Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:38Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:38Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:09Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:36Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4336, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4323, "optime" : { "t" : 1358382157000, "i" : 2587 }, "optimeDate" : ISODate("2013-01-17T00:22:37Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:37Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:38Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:38Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4323, "optime" : { "t" : 1358382158000, "i" : 1787 }, "optimeDate" : ISODate("2013-01-17T00:22:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4323, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:37Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4335, "optime" : { "t" : 1358382158000, "i" : 1924 }, "optimeDate" : ISODate("2013-01-17T00:22:38Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:22:38.820 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:22:40.244 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:22:40.820 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4341, "optime" : { "t" : 1358382162000, "i" : 3204 }, "optimeDate" : ISODate("2013-01-17T00:22:42Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4330, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4330, "optime" : { "t" : 1358382158000, "i" : 1924 }, "optimeDate" : ISODate("2013-01-17T00:22:38Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:42Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:42Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4340, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4327, "optime" : { "t" : 1358382161000, "i" : 909 }, "optimeDate" : ISODate("2013-01-17T00:22:41Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:41Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:42Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4327, "optime" : { "t" : 1358382162000, "i" : 1773 }, "optimeDate" : ISODate("2013-01-17T00:22:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4327, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:41Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4339, "optime" : { "t" : 1358382162000, "i" : 3204 }, "optimeDate" : ISODate("2013-01-17T00:22:42Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:22:42.821 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:22:44.821 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:22:45.588 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4345, "optime" : { "t" : 1358382166000, "i" : 5164 }, "optimeDate" : ISODate("2013-01-17T00:22:46Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4334, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4334, "optime" : { "t" : 1358382166000, "i" : 457 }, "optimeDate" : ISODate("2013-01-17T00:22:46Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:46Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:46Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4344, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4331, "optime" : { "t" : 1358382162000, "i" : 3204 }, "optimeDate" : ISODate("2013-01-17T00:22:42Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:45Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:46Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4331, "optime" : { "t" : 1358382166000, "i" : 1996 }, "optimeDate" : ISODate("2013-01-17T00:22:46Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:46Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4331, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:45Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4343, "optime" : { "t" : 1358382166000, "i" : 5164 }, "optimeDate" : ISODate("2013-01-17T00:22:46Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31001| Wed Jan 16 19:22:46.821 [conn258] end connection 127.0.0.2:57575 (2 connections now open) m31001| Wed Jan 16 19:22:46.822 [initandlisten] connection accepted from 127.0.0.2:37453 #260 (3 connections now open) m31000| Wed Jan 16 19:22:46.822 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:22:48.822 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:22:50.245 [conn154] end connection 127.0.0.4:51041 (4 connections now open) m31000| Wed Jan 16 19:22:50.246 [initandlisten] connection accepted from 127.0.0.4:33007 #155 (5 connections now open) m31002| Wed Jan 16 19:22:50.246 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4349, "optime" : { "t" : 1358382170000, "i" : 6854 }, "optimeDate" : ISODate("2013-01-17T00:22:50Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4338, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4338, "optime" : { "t" : 1358382166000, "i" : 5164 }, "optimeDate" : ISODate("2013-01-17T00:22:46Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:48Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:50Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:50Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4348, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4335, "optime" : { "t" : 1358382166000, "i" : 5164 }, "optimeDate" : ISODate("2013-01-17T00:22:46Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:49Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:50Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:50Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4335, "optime" : { "t" : 1358382170000, "i" : 2127 }, "optimeDate" : ISODate("2013-01-17T00:22:50Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4335, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:50Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4347, "optime" : { "t" : 1358382170000, "i" : 6117 }, "optimeDate" : ISODate("2013-01-17T00:22:50Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:22:50.823 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:22:52.823 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:22:53.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:22:54.823 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4353, "optime" : { "t" : 1358382174000, "i" : 7973 }, "optimeDate" : ISODate("2013-01-17T00:22:54Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4342, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4342, "optime" : { "t" : 1358382170000, "i" : 6854 }, "optimeDate" : ISODate("2013-01-17T00:22:50Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:53Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:54Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:54Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:54Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4352, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4339, "optime" : { "t" : 1358382170000, "i" : 6854 }, "optimeDate" : ISODate("2013-01-17T00:22:50Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:53Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:54Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4339, "optime" : { "t" : 1358382174000, "i" : 2232 }, "optimeDate" : ISODate("2013-01-17T00:22:54Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:54Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4339, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:52Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:53Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4351, "optime" : { "t" : 1358382174000, "i" : 7973 }, "optimeDate" : ISODate("2013-01-17T00:22:54Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:22:55.589 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:22:56.824 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:22:58.824 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4358, "optime" : { "t" : 1358382179000, "i" : 767 }, "optimeDate" : ISODate("2013-01-17T00:22:59Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4347, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4347, "optime" : { "t" : 1358382178000, "i" : 4559 }, "optimeDate" : ISODate("2013-01-17T00:22:58Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:58Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:59Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4357, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4344, "optime" : { "t" : 1358382174000, "i" : 7973 }, "optimeDate" : ISODate("2013-01-17T00:22:54Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:57Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:58Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:22:59Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4344, "optime" : { "t" : 1358382178000, "i" : 1936 }, "optimeDate" : ISODate("2013-01-17T00:22:58Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:58Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:22:58Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4344, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:57Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4356, "optime" : { "t" : 1358382179000, "i" : 767 }, "optimeDate" : ISODate("2013-01-17T00:22:59Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:23:00.247 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:23:00.824 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:23:02.824 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4362, "optime" : { "t" : 1358382183000, "i" : 2516 }, "optimeDate" : ISODate("2013-01-17T00:23:03Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4351, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4351, "optime" : { "t" : 1358382179000, "i" : 767 }, "optimeDate" : ISODate("2013-01-17T00:22:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:00Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:02Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:03Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:02Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4361, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4348, "optime" : { "t" : 1358382179000, "i" : 767 }, "optimeDate" : ISODate("2013-01-17T00:22:59Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:01Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:03Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:03Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4348, "optime" : { "t" : 1358382182000, "i" : 1080 }, "optimeDate" : ISODate("2013-01-17T00:23:02Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4348, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:02Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4360, "optime" : { "t" : 1358382183000, "i" : 2077 }, "optimeDate" : ISODate("2013-01-17T00:23:03Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:23:03.405 [conn261] end connection 127.0.0.3:48572 (3 connections now open) m31002| Wed Jan 16 19:23:03.405 [initandlisten] connection accepted from 127.0.0.3:38615 #262 (4 connections now open) m31001| Wed Jan 16 19:23:03.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31000| Wed Jan 16 19:23:04.825 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:23:05.590 [rsMgr] replSet info two primaries (transiently) m31002| Wed Jan 16 19:23:06.095 [conn260] end connection 127.0.0.2:41238 (3 connections now open) m31002| Wed Jan 16 19:23:06.596 [initandlisten] connection accepted from 127.0.0.2:35089 #263 (4 connections now open) m31000| Wed Jan 16 19:23:06.681 [journal] old journal file will be removed: /data/db/testReplSet-0/journal/j._6 m31000| Wed Jan 16 19:23:06.769 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:301690 150ms m31000| Wed Jan 16 19:23:06.825 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4366, "optime" : { "t" : 1358382187000, "i" : 3461 }, "optimeDate" : ISODate("2013-01-17T00:23:07Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4355, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4355, "optime" : { "t" : 1358382186000, "i" : 2518 }, "optimeDate" : ISODate("2013-01-17T00:23:06Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:06Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:07Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4365, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4352, "optime" : { "t" : 1358382187000, "i" : 3385 }, "optimeDate" : ISODate("2013-01-17T00:23:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:07Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:07Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4352, "optime" : { "t" : 1358382183000, "i" : 2516 }, "optimeDate" : ISODate("2013-01-17T00:23:03Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:06Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:06Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4352, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:05Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:07Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4364, "optime" : { "t" : 1358382187000, "i" : 3385 }, "optimeDate" : ISODate("2013-01-17T00:23:07Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:23:08.825 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:23:10.249 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:23:10.591 [conn259] end connection 127.0.0.4:47623 (2 connections now open) m31000| Wed Jan 16 19:23:10.826 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:23:11.091 [initandlisten] connection accepted from 127.0.0.4:35001 #261 (3 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4370, "optime" : { "t" : 1358382191000, "i" : 5253 }, "optimeDate" : ISODate("2013-01-17T00:23:11Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4359, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4359, "optime" : { "t" : 1358382187000, "i" : 3461 }, "optimeDate" : ISODate("2013-01-17T00:23:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:08Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:10Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:11Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:22:40Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:10Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4369, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4356, "optime" : { "t" : 1358382191000, "i" : 3603 }, "optimeDate" : ISODate("2013-01-17T00:23:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:11Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:11Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4356, "optime" : { "t" : 1358382187000, "i" : 3461 }, "optimeDate" : ISODate("2013-01-17T00:23:07Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:10Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4356, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:10Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:11Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4368, "optime" : { "t" : 1358382191000, "i" : 5253 }, "optimeDate" : ISODate("2013-01-17T00:23:11Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:23:12.826 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:23:14.826 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4374, "optime" : { "t" : 1358382195000, "i" : 6769 }, "optimeDate" : ISODate("2013-01-17T00:23:15Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4363, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4363, "optime" : { "t" : 1358382191000, "i" : 5253 }, "optimeDate" : ISODate("2013-01-17T00:23:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:14Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:15Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:14Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4373, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4360, "optime" : { "t" : 1358382195000, "i" : 3579 }, "optimeDate" : ISODate("2013-01-17T00:23:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:15Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:15Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4360, "optime" : { "t" : 1358382191000, "i" : 5253 }, "optimeDate" : ISODate("2013-01-17T00:23:11Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:14Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4360, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:15Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4372, "optime" : { "t" : 1358382195000, "i" : 6769 }, "optimeDate" : ISODate("2013-01-17T00:23:15Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31002| Wed Jan 16 19:23:16.092 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:23:16.827 [conn260] end connection 127.0.0.2:37453 (2 connections now open) m31001| Wed Jan 16 19:23:16.827 [initandlisten] connection accepted from 127.0.0.2:59855 #262 (3 connections now open) m31000| Wed Jan 16 19:23:16.827 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:23:18.828 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4378, "optime" : { "t" : 1358382199000, "i" : 7775 }, "optimeDate" : ISODate("2013-01-17T00:23:19Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4367, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4367, "optime" : { "t" : 1358382199000, "i" : 512 }, "optimeDate" : ISODate("2013-01-17T00:23:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:18Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:19Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:18Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4377, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4364, "optime" : { "t" : 1358382199000, "i" : 3201 }, "optimeDate" : ISODate("2013-01-17T00:23:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:19Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:19Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4364, "optime" : { "t" : 1358382195000, "i" : 6769 }, "optimeDate" : ISODate("2013-01-17T00:23:15Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:19Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4364, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:18Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:19Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4376, "optime" : { "t" : 1358382199000, "i" : 7775 }, "optimeDate" : ISODate("2013-01-17T00:23:19Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:23:20.251 [conn155] end connection 127.0.0.4:33007 (4 connections now open) m31000| Wed Jan 16 19:23:20.251 [initandlisten] connection accepted from 127.0.0.4:41880 #156 (5 connections now open) m31000| Wed Jan 16 19:23:20.828 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:23:21.093 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:23:22.828 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:23:23.726 [conn1] insert foo.bar ninserted:1 keyUpdates:0 locks(micros) w:279483 139ms STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4383, "optime" : { "t" : 1358382204000, "i" : 69 }, "optimeDate" : ISODate("2013-01-17T00:23:24Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4372, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4372, "optime" : { "t" : 1358382199000, "i" : 7775 }, "optimeDate" : ISODate("2013-01-17T00:23:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:21Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:22Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:24Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:22Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4382, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4369, "optime" : { "t" : 1358382203000, "i" : 4055 }, "optimeDate" : ISODate("2013-01-17T00:23:23Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:23Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:23Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:24Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4369, "optime" : { "t" : 1358382199000, "i" : 7775 }, "optimeDate" : ISODate("2013-01-17T00:23:19Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:22Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4369, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:23Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4381, "optime" : { "t" : 1358382203000, "i" : 9520 }, "optimeDate" : ISODate("2013-01-17T00:23:23Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:23:24.829 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:23:25.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:23:26.093 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:23:26.829 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4387, "optime" : { "t" : 1358382208000, "i" : 1392 }, "optimeDate" : ISODate("2013-01-17T00:23:28Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4376, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4376, "optime" : { "t" : 1358382204000, "i" : 69 }, "optimeDate" : ISODate("2013-01-17T00:23:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:26Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:28Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4386, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4373, "optime" : { "t" : 1358382207000, "i" : 3018 }, "optimeDate" : ISODate("2013-01-17T00:23:27Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:27Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:28Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4373, "optime" : { "t" : 1358382204000, "i" : 69 }, "optimeDate" : ISODate("2013-01-17T00:23:24Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:26Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:26Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4373, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:25Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:27Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4385, "optime" : { "t" : 1358382208000, "i" : 1392 }, "optimeDate" : ISODate("2013-01-17T00:23:28Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:23:28.830 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:23:30.830 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:23:31.094 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4391, "optime" : { "t" : 1358382212000, "i" : 2545 }, "optimeDate" : ISODate("2013-01-17T00:23:32Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4380, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:30Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4380, "optime" : { "t" : 1358382211000, "i" : 3777 }, "optimeDate" : ISODate("2013-01-17T00:23:31Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:31Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:32Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:32Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:30Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4390, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4377, "optime" : { "t" : 1358382211000, "i" : 1686 }, "optimeDate" : ISODate("2013-01-17T00:23:31Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:31Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:32Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4377, "optime" : { "t" : 1358382212000, "i" : 2271 }, "optimeDate" : ISODate("2013-01-17T00:23:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:32Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4377, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:30Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:31Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4389, "optime" : { "t" : 1358382212000, "i" : 2545 }, "optimeDate" : ISODate("2013-01-17T00:23:32Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:23:32.830 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:23:33.410 [conn262] end connection 127.0.0.3:38615 (3 connections now open) m31002| Wed Jan 16 19:23:33.410 [initandlisten] connection accepted from 127.0.0.3:55443 #264 (4 connections now open) m31000| Wed Jan 16 19:23:34.831 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:23:35.627 [rsHealthPoll] couldn't connect to 127.0.0.2:31000: couldn't connect to server 127.0.0.2:31000 m31002| Wed Jan 16 19:23:36.095 [rsMgr] replSet info two primaries (transiently) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4395, "optime" : { "t" : 1358382216000, "i" : 4069 }, "optimeDate" : ISODate("2013-01-17T00:23:36Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4384, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:34Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4384, "optime" : { "t" : 1358382212000, "i" : 2545 }, "optimeDate" : ISODate("2013-01-17T00:23:32Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:33Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:36Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:36Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:34Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4394, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4381, "optime" : { "t" : 1358382215000, "i" : 615 }, "optimeDate" : ISODate("2013-01-17T00:23:35Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:35Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:36Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:36Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4381, "optime" : { "t" : 1358382216000, "i" : 2758 }, "optimeDate" : ISODate("2013-01-17T00:23:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:36Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4381, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:35Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4393, "optime" : { "t" : 1358382216000, "i" : 3829 }, "optimeDate" : ISODate("2013-01-17T00:23:36Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:23:36.831 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31000| Wed Jan 16 19:23:38.832 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4399, "optime" : { "t" : 1358382220000, "i" : 5023 }, "optimeDate" : ISODate("2013-01-17T00:23:40Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4388, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:38Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4388, "optime" : { "t" : 1358382216000, "i" : 4069 }, "optimeDate" : ISODate("2013-01-17T00:23:36Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:40Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:40Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:38Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4398, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4385, "optime" : { "t" : 1358382219000, "i" : 38 }, "optimeDate" : ISODate("2013-01-17T00:23:39Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:39Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:40Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4385, "optime" : { "t" : 1358382220000, "i" : 2098 }, "optimeDate" : ISODate("2013-01-17T00:23:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:40Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4385, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:38Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:39Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4397, "optime" : { "t" : 1358382220000, "i" : 5023 }, "optimeDate" : ISODate("2013-01-17T00:23:40Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:23:40.832 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:23:41.095 [rsMgr] replSet info two primaries (transiently) m31000| Wed Jan 16 19:23:42.832 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:23:44.101 [conn263] end connection 127.0.0.2:35089 (3 connections now open) m31002| Wed Jan 16 19:23:44.601 [initandlisten] connection accepted from 127.0.0.2:59359 #265 (4 connections now open) STATUS FROM : 127.0.0.2:31000 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4403, "optime" : { "t" : 1358382224000, "i" : 6439 }, "optimeDate" : ISODate("2013-01-17T00:23:44Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4392, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:42Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4392, "optime" : { "t" : 1358382224000, "i" : 5211 }, "optimeDate" : ISODate("2013-01-17T00:23:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:44Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:44Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:13Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:42Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4402, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4389, "optime" : { "t" : 1358382220000, "i" : 5023 }, "optimeDate" : ISODate("2013-01-17T00:23:40Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:43Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:43Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:44Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4389, "optime" : { "t" : 1358382224000, "i" : 2220 }, "optimeDate" : ISODate("2013-01-17T00:23:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:44Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:44Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4389, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:43Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4401, "optime" : { "t" : 1358382224000, "i" : 5531 }, "optimeDate" : ISODate("2013-01-17T00:23:44Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } } m31000| Wed Jan 16 19:23:44.833 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31002| Wed Jan 16 19:23:46.096 [rsMgr] replSet info two primaries (transiently) m31001| Wed Jan 16 19:23:46.833 [conn262] end connection 127.0.0.2:59855 (2 connections now open) m31001| Wed Jan 16 19:23:46.833 [initandlisten] connection accepted from 127.0.0.2:36653 #263 (3 connections now open) m31000| Wed Jan 16 19:23:46.833 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down m31001| Wed Jan 16 19:23:48.596 [conn261] end connection 127.0.0.4:35001 (2 connections now open) STATUS FROM : 127.0.0.2:31000 m31000| Wed Jan 16 19:23:48.834 [rsHealthPoll] replset info 127.0.0.3:31001 thinks that we are down { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4407, "optime" : { "t" : 1358382228000, "i" : 7784 }, "optimeDate" : ISODate("2013-01-17T00:23:48Z"), "self" : true }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4396, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4396, "optime" : { "t" : 1358382224000, "i" : 6439 }, "optimeDate" : ISODate("2013-01-17T00:23:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:46Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:48Z"), "pingMs" : 500 } ], "ok" : 1 } STATUS FROM : 127.0.0.3:31001 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:48Z"), "myState" : 1, "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 0, "state" : 8, "stateStr" : "(not reachable/healthy)", "uptime" : 0, "optime" : { "t" : 1358377825000, "i" : 1 }, "optimeDate" : ISODate("2013-01-16T23:10:25Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:48Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4406, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "self" : true }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4393, "optime" : { "t" : 1358382224000, "i" : 6439 }, "optimeDate" : ISODate("2013-01-17T00:23:44Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:47Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 } ], "ok" : 1 } STATUS FROM : 127.0.0.4:31002 { "set" : "testReplSet", "date" : ISODate("2013-01-17T00:23:48Z"), "myState" : 2, "syncingTo" : "127.0.0.2:31000", "members" : [ { "_id" : 0, "name" : "127.0.0.2:31000", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4393, "optime" : { "t" : 1358382228000, "i" : 1859 }, "optimeDate" : ISODate("2013-01-17T00:23:48Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:48Z"), "lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"), "pingMs" : 0 }, { "_id" : 1, "name" : "127.0.0.3:31001", "health" : 1, "state" : 1, "stateStr" : "PRIMARY", "uptime" : 4393, "optime" : { "t" : 1358378165000, "i" : 6931 }, "optimeDate" : ISODate("2013-01-16T23:16:05Z"), "lastHeartbeat" : ISODate("2013-01-17T00:23:45Z"), "lastHeartbeatRecv" : ISODate("2013-01-17T00:23:47Z"), "pingMs" : 500 }, { "_id" : 2, "name" : "127.0.0.4:31002", "health" : 1, "state" : 2, "stateStr" : "SECONDARY", "uptime" : 4405, "optime" : { "t" : 1358382228000, "i" : 7583 }, "optimeDate" : ISODate("2013-01-17T00:23:48Z"), "self" : true } ], "ok" : 1 } { "roots" : [ "127.0.0.2:31000", "127.0.0.3:31001" ], "graph" : { "127.0.0.2:31000" : [ "127.0.0.4:31002" ] } }