2017-05-12T16:11:22.765-0700 I SHARDING [conn1249] Starting chunk migration ns: agent.pageloadHarEntries, [{ _id: "a23036:108:1483038000:1483038060" }, { _id: "a23036:108:1483801200:1483801260" }), fromShard: rs2, toShard: rs1 with expected collection version 12468|43||57fd4a8a4d8a45221b5f0c2e 2017-05-12T16:11:22.765-0700 I SHARDING [conn1249] MetadataLoader loading chunks for agent.pageloadHarEntries based on: 12468|43||57fd4a8a4d8a45221b5f0c2e 2017-05-12T16:11:22.794-0700 I SHARDING [conn1249] about to log metadata event into changelog: { _id: "msc2-1.stg.dc1.thousandeyes.com-2017-05-12T16:11:22.794-0700-5916411acdb45b6c28ab5408", server: "msc2-1.stg.dc1.thousandeyes.com", clientAddr: "10.0.21.22:38251", time: new Date(1494630682794), what: "moveChunk.start", ns: "agent.pageloadHarEntries", details: { min: { _id: "a23036:108:1483038000:1483038060" }, max: { _id: "a23036:108:1483801200:1483801260" }, from: "rs2", to: "rs1" } } 2017-05-12T16:11:22.872-0700 I COMMAND [conn1622] command agent.pageloadHarEntries command: listIndexes { listIndexes: "pageloadHarEntries", cursor: {} } numYields:0 reslen:433 locks:{ Global: { acquireCount: { r: 2 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { r: 1 } } } protocol:op_command 0ms 2017-05-12T16:11:22.873-0700 I SHARDING [conn1249] moveChunk data transfer progress: { active: true, sessionId: "rs2_rs1_59164198cdb45b6c28ab674e", ns: "agent.pageloadHarEntries", from: "rs2/msc2-1.stg:27017,msc2-2.stg:27017", min: { _id: "a23036:108:1483038000:1483038060" }, max: { _id: "a23036:108:1483801200:1483801260" }, shardKeyPattern: { _id: 1.0 }, state: "ready", counts: { cloned: 0, clonedBytes: 0, catchup: 0, steady: 0 }, ok: 1.0 } mem used: 0 documents remaining to clone: 0 2017-05-12T16:11:22.875-0700 I SHARDING [conn1249] moveChunk data transfer progress: { active: true, sessionId: "rs2_rs1_59164198cdb45b6c28ab674e", ns: "agent.pageloadHarEntries", from: "rs2/msc2-1.stg:27017,msc2-2.stg:27017", min: { _id: "a23036:108:1483038000:1483038060" }, max: { _id: "a23036:108:1483801200:1483801260" }, shardKeyPattern: { _id: 1.0 }, state: "ready", counts: { cloned: 0, clonedBytes: 0, catchup: 0, steady: 0 }, ok: 1.0 } mem used: 0 documents remaining to clone: 0 2017-05-12T16:11:22.880-0700 I SHARDING [conn1249] moveChunk data transfer progress: { active: true, sessionId: "rs2_rs1_59164198cdb45b6c28ab674e", ns: "agent.pageloadHarEntries", from: "rs2/msc2-1.stg:27017,msc2-2.stg:27017", min: { _id: "a23036:108:1483038000:1483038060" }, max: { _id: "a23036:108:1483801200:1483801260" }, shardKeyPattern: { _id: 1.0 }, state: "ready", counts: { cloned: 0, clonedBytes: 0, catchup: 0, steady: 0 }, ok: 1.0 } mem used: 0 documents remaining to clone: 0 2017-05-12T16:11:22.887-0700 I COMMAND [conn1622] command admin.$cmd command: _migrateClone { _migrateClone: "agent.pageloadHarEntries", sessionId: "rs2_rs1_59164198cdb45b6c28ab674e" } numYields:0 reslen:148 locks:{ Global: { acquireCount: { r: 2 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { r: 1 } } } protocol:op_command 0ms 2017-05-12T16:11:22.887-0700 I COMMAND [conn1622] command admin.$cmd command: _transferMods { _transferMods: "agent.pageloadHarEntries", sessionId: "rs2_rs1_59164198cdb45b6c28ab674e" } numYields:0 reslen:148 locks:{ Global: { acquireCount: { r: 2 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { r: 1 } } } protocol:op_command 0ms 2017-05-12T16:11:22.888-0700 I COMMAND [conn1622] command admin.$cmd command: _transferMods { _transferMods: "agent.pageloadHarEntries", sessionId: "rs2_rs1_59164198cdb45b6c28ab674e" } numYields:0 reslen:148 locks:{ Global: { acquireCount: { r: 2 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { r: 1 } } } protocol:op_command 0ms 2017-05-12T16:11:22.889-0700 I SHARDING [conn1249] moveChunk data transfer progress: { active: true, sessionId: "rs2_rs1_59164198cdb45b6c28ab674e", ns: "agent.pageloadHarEntries", from: "rs2/msc2-1.stg:27017,msc2-2.stg:27017", min: { _id: "a23036:108:1483038000:1483038060" }, max: { _id: "a23036:108:1483801200:1483801260" }, shardKeyPattern: { _id: 1.0 }, state: "steady", counts: { cloned: 0, clonedBytes: 0, catchup: 0, steady: 0 }, ok: 1.0 } mem used: 0 documents remaining to clone: 0 2017-05-12T16:11:22.940-0700 I COMMAND [conn1622] command admin.$cmd command: _transferMods { _transferMods: "agent.pageloadHarEntries", sessionId: "rs2_rs1_59164198cdb45b6c28ab674e" } numYields:0 reslen:148 locks:{ Global: { acquireCount: { r: 2 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { r: 1 } } } protocol:op_command 0ms [TONS OF _transferMods COMMANDS] 2017-05-12T16:13:43.419-0700 I SHARDING [conn1249] about to log metadata event into changelog: { _id: "msc2-1.stg.dc1.thousandeyes.com-2017-05-12T16:13:43.419-0700-591641a7cdb45b6c28ab6e18", server: "msc2-1.stg.dc1.thousandeyes.com", clientAddr: "10.0.21.22:38251", time: new Date(1494630823419), what: "moveChunk.error", ns: "agent.pageloadHarEntries", details: { min: { _id: "a23036:108:1483038000:1483038060" }, max: { _id: "a23036:108:1483801200:1483801260" }, from: "rs2", to: "rs1" } } 2017-05-12T16:13:43.419-0700 I COMMAND [conn1622] command admin.$cmd command: _transferMods { _transferMods: "agent.pageloadHarEntries", sessionId: "rs2_rs1_59164198cdb45b6c28ab674e" } numYields:0 reslen:148 locks:{ Global: { acquireCount: { r: 2 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { r: 1 } } } protocol:op_command 0ms 2017-05-12T16:13:43.430-0700 I COMMAND [conn1622] command admin.$cmd command: _transferMods { _transferMods: "agent.pageloadHarEntries", sessionId: "rs2_rs1_59164198cdb45b6c28ab674e" } numYields:0 reslen:148 locks:{ Global: { acquireCount: { r: 2 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { r: 1 } } } protocol:op_command 0ms 2017-05-12T16:13:43.435-0700 I SHARDING [conn1249] about to log metadata event into changelog: { _id: "msc2-1.stg.dc1.thousandeyes.com-2017-05-12T16:13:43.435-0700-591641a7cdb45b6c28ab6e1c", server: "msc2-1.stg.dc1.thousandeyes.com", clientAddr: "10.0.21.22:38251", time: new Date(1494630823435), what: "moveChunk.from", ns: "agent.pageloadHarEntries", details: { min: { _id: "a23036:108:1483038000:1483038060" }, max: { _id: "a23036:108:1483801200:1483801260" }, step 1 of 7: 0, step 2 of 7: 29, step 3 of 7: 54, step 4 of 7: 18, to: "rs1", from: "rs2", note: "aborted" } } 2017-05-12T16:13:43.474-0700 I COMMAND [conn1249] command admin.$cmd appName: "MongoDB Shell" command: moveChunk { moveChunk: "agent.pageloadHarEntries", shardVersion: [ Timestamp 12468000|43, ObjectId('57fd4a8a4d8a45221b5f0c2e') ], epoch: ObjectId('57fd4a8a4d8a45221b5f0c2e'), configdb: "csReplSet/mongoc1.stg.dc1.thousandeyes.com:27019,mongoc2.stg.dc1.thousandeyes.com:27031,mongoc3.stg.dc1.thousandeyes.com:27032", fromShard: "rs2", toShard: "rs1", min: { _id: "a23036:108:1483038000:1483038060" }, max: { _id: "a23036:108:1483801200:1483801260" }, chunkVersion: [ Timestamp 12468000|1, ObjectId('57fd4a8a4d8a45221b5f0c2e') ], maxChunkSizeBytes: 67108864, secondaryThrottle: false, waitForDelete: false, takeDistLock: false } exception: waiting for replication timed out code:64 numYields:0 reslen:243 locks:{ Global: { acquireCount: { r: 13, w: 7 } }, Database: { acquireCount: { r: 3, w: 5, W: 2 } }, Collection: { acquireCount: { r: 3, W: 3 } }, Metadata: { acquireCount: { w: 2 } }, oplog: { acquireCount: { w: 2 } } } protocol:op_command 15185ms