Details
Description
Hi,
v2.8.0-rc5 crashes during a large data import to a cluster of 3 mongod instances with the error message:
"write results unavailable from xxx.xxx.xx:27011 :: caused by :: Location 17255 error receiving write command response, possible socket exception
|
The batch insert is failing on the client side but the server log is also crashing.
This occurs when there are 4 parallel imports being run on each of the 3 mongos instances.
mongod log error:
2015-01-28T08:40:01.421+0100 I WRITE [conn78] insert COLLECTION-tigercluster.COLLECTION query: { _id: "54c89242228b39f12990b57c", y: 1, o: 10, t: 1395314451000, p: { g: [ 5.9130544653739, 51.650652567424 ], h: 154.0, z: 13.0, v: 3916, q: 2, a: 0 }, r: "B61FA96E-9DB0-4EE2-A40C-858E6C8013A4" } ninserted:1 keyUpdates:0 writeConflicts:0 numYields:0 850ms
|
2015-01-28T08:40:01.432+0100 I SHARDING [conn70] moveChunk data transfer progress: { active: true, ns: "COLLECTION-tigercluster.COLLECTION", from: "tigersrd3_1/xxx.xxx.xx:270xx", min: { r: "A50BCF9C-EFDF-40FF-B47C-42C728D05762" }, max: { r: "A52F12EE-C721-487F-BABF-914A70930B66" }, shardKeyPattern: { r: 1.0 }, state: "clone", counts: { cloned: 85649, clonedBytes: 16187661, catchup: 0, steady: 0 }, ok: 1.0 } my mem used: 0
|
2015-01-28T08:40:01.434+0100 I - [conn76] Invariant failure: ret resulted in status UnknownError 2: No such file or directory at src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp 438
|
2015-01-28T08:40:01.432+0100 I WRITE [conn68] insert COLLECTION-tigercluster.COLLECTION query: { _id: "54c89242a3b8d4021e42387d", y: 1, o: 10, t: 1393952186000, p: { g: [ 5.4865192694804, 52.164699210253 ], h: 97.0, z: 7.0, v: 3277, q: 2, a: 0 }, r: "D857EB1F-E235-4C58-9283-AA8C4013C003" } ninserted:1 keyUpdates:0 writeConflicts:0 numYields:0 878ms
|
2015-01-28T08:40:01.440+0100 I WRITE [conn66] insert COLLECTION-tigercluster.COLLECTION query: { _id: "54c8923fcdb479b37f68d14e", y: 1, o: 10, t: 1393635113000, p: { g: [ 5.047504050466, 52.070182255382 ], h: 104.0, z: -4.0, v: 2722, q: 2, a: 0 }, r: "CD9A2E28-DA7C-4191-8B3F-51EEAFF7919F" } ninserted:1 keyUpdates:0 writeConflicts:0 numYields:0 875ms
|
2015-01-28T08:40:01.440+0100 I WRITE [conn63] insert COLLECTION-tigercluster.COLLECTION query: { _id: "54c89240973f4164a0cff8da", y: 1, o: 10, t: 1394193599000, p: { g: [ 4.8112359278986, 52.316372955215 ], h: 328.0, z: 2.0, v: 2805, q: 2, a: 0 }, r: "B58B1CF8-3F50-40F1-846A-C97AF0957F45" } ninserted:1 keyUpdates:0 writeConflicts:0 numYields:0 885ms
|
2015-01-28T08:40:01.441+0100 I WRITE [conn82] insert COLLECTION-tigercluster.COLLECTION query: { _id: "54c892422e2461dc2ba41b92", y: 1, o: 10, t: 1393398599000, p: { g: [ 5.1026667810514, 51.974993455037 ], h: 129.0, z: 3.0, v: 3388, q: 2, a: 0 }, r: "F1BDE97C-5018-45F3-97B0-B61BF1CF1512" } ninserted:1 keyUpdates:0 writeConflicts:0 numYields:0 880ms
|
2015-01-28T08:40:01.446+0100 I WRITE [conn87] insert COLLECTION-tigercluster.COLLECTION query: { _id: "54c892421ccd143413ee1d5f", y: 1, o: 10, t: 1394605462000, p: { g: [ 5.0560193578298, 52.108511417551 ], h: 314.0, z: -2.0, v: 2722, q: 2, a: 0 }, r: "D638759B-3A7A-43EB-83F9-26F8C14E60F6" } ninserted:1 keyUpdates:0 writeConflicts:0 numYields:0 879ms
|
2015-01-28T08:40:01.457+0100 I CONTROL [conn76]
|
0xf25749 0xecf571 0xeb629a 0xd46704 0xd47bfe 0xd44f4d 0x8f055c 0xc1c4ac 0xc1a483 0x98e9fd 0x98f11a 0x9908b4 0x990fb5 0x99306d 0x9b31c4 0x9b4103 0x9b4bbb 0xb80ea5 0xa941fa 0x7e5320 0xee37d1 0x7fd727974e9a 0x7fd726a842ed
|
----- BEGIN BACKTRACE -----
|
{"backtrace":[{"b":"400000","o":"B25749"},{"b":"400000","o":"ACF571"},{"b":"400000","o":"AB629A"},{"b":"400000","o":"946704"},{"b":"400000","o":"947BFE"},{"b":"400000","o":"944F4D"},{"b":"400000","o":"4F055C"},{"b":"400000","o":"81C4AC"},{"b":"400000","o":"81A483"},{"b":"400000","o":"58E9FD"},{"b":"400000","o":"58F11A"},{"b":"400000","o":"5908B4"},{"b":"400000","o":"590FB5"},{"b":"400000","o":"59306D"},{"b":"400000","o":"5B31C4"},{"b":"400000","o":"5B4103"},{"b":"400000","o":"5B4BBB"},{"b":"400000","o":"780EA5"},{"b":"400000","o":"6941FA"},{"b":"400000","o":"3E5320"},{"b":"400000","o":"AE37D1"},{"b":"7FD72796D000","o":"7E9A"},{"b":"7FD726990000","o":"F42ED"}],"processInfo":{ "mongodbVersion" : "2.8.0-rc5", "gitVersion" : "74b351de21c84438b12a83b28e155f5e69e3c1eb", "uname" : { "sysname" : "Linux", "release" : "3.2.0-74-generic", "version" : "#109-Ubuntu SMP Tue Dec 9 16:45:49 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000" }, { "b" : "7FFF120FF000", "elfType" : 3 }, { "b" : "7FD72796D000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3 }, { "b" : "7FD727765000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3 }, { "b" : "7FD727561000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3 }, { "b" : "7FD727261000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3 }, { "b" : "7FD726F65000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3 }, { "b" : "7FD726D4F000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3 }, { "b" : "7FD726990000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3 }, { "b" : "7FD727B8A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3 } ] }}
|
mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf25749]
|
mongod(_ZN5mongo10logContextEPKc+0xE1) [0xecf571]
|
mongod(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0xDA) [0xeb629a]
|
mongod(_ZN5mongo21WiredTigerRecordStore20cappedDeleteAsNeededEPNS_16OperationContextERKNS_8RecordIdE+0x5F4) [0xd46704]
|
mongod(_ZN5mongo21WiredTigerRecordStore12insertRecordEPNS_16OperationContextEPKcib+0x1DE) [0xd47bfe]
|
mongod(_ZN5mongo21WiredTigerRecordStore12insertRecordEPNS_16OperationContextEPKNS_9DocWriterEb+0x8D) [0xd44f4d]
|
mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextEPKNS_9DocWriterEb+0x5C) [0x8f055c]
|
mongod(+0x81C4AC) [0xc1c4ac]
|
mongod(_ZN5mongo4repl5logOpEPNS_16OperationContextEPKcS4_RKNS_7BSONObjEPS5_Pbb+0xA3) [0xc1a483]
|
mongod(_ZN5mongo18WriteBatchExecutor13execOneInsertEPNS0_16ExecInsertsStateEPPNS_16WriteErrorDetailE+0xEED) [0x98e9fd]
|
mongod(_ZN5mongo18WriteBatchExecutor11execInsertsERKNS_21BatchedCommandRequestEPSt6vectorIPNS_16WriteErrorDetailESaIS6_EE+0x25A) [0x98f11a]
|
mongod(_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestEPSt6vectorIPNS_19BatchedUpsertDetailESaIS6_EEPS4_IPNS_16WriteErrorDetailESaISB_EE+0x34) [0x9908b4]
|
mongod(_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x395) [0x990fb5]
|
mongod(_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x15D) [0x99306d]
|
mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9b31c4]
|
mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC13) [0x9b4103]
|
mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9b4bbb]
|
mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_b+0x755) [0xb80ea5]
|
mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortEb+0xB0A) [0xa941fa]
|
mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xE0) [0x7e5320]
|
mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x321) [0xee37d1]
|
libpthread.so.0(+0x7E9A) [0x7fd727974e9a]
|
libc.so.6(clone+0x6D) [0x7fd726a842ed]
|
----- END BACKTRACE -----
|
2015-01-28T08:40:01.457+0100 I - [conn76]
|
Attachments
Issue Links
- related to
-
SERVER-17141 session->truncate failed with File Not Found in WiredTigerRecordStoreThread for local.oplog.rs
-
- Closed
-