Details
-
Bug
-
Resolution: Done
-
Critical - P2
-
None
-
None
-
None
-
ALL
Description
Hi,
I have a 3.0.14 mmapv1 server repeatedly crashing with the following stack trace
w: 1 } }, Collection: { acquireCount: { W: 1 } } } 109ms
|
2016-11-17T13:09:43.840+0000 I STORAGE [conn27] _getOpenFile() invalid file index requested 1832673968
|
2016-11-17T13:09:43.840+0000 I - [conn27] Invariant failure false src/mongo/db/storage/mmap_v1/mmap_v1_extent_manager.cpp 201
|
2016-11-17T13:09:43.865+0000 I CONTROL [conn27]
|
0xfc2852 0xf5d8b9 0xf40c89 0xd9509f 0xd95246 0xd9527f 0xda6516 0xda7523 0xd9c12b 0xd9c2d9 0x9303f1 0x930f9c 0xa6435c 0xa65094 0xc0dd32 0xc0e04b 0xc0e72d 0x9d7499 0x9d8cf5 0x9d9325 0x9
|
dc00c 0x9fed91 0x9ffd9c 0xa00a6b 0xbdb12b 0xae9aaa 0x823335 0xf72189 0x3766607aa1 0x37662e8aad
|
----- BEGIN BACKTRACE -----
|
{"backtrace":[{"b":"400000","o":"BC2852","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"B5D8B9","s":"_ZN5mongo10logContextEPKc"},{"b":"400000","o":"B40C89","s":"_ZN5mongo15in
|
variantFailedEPKcS1_j"},{"b":"400000","o":"99509F","s":"_ZNK5mongo19MmapV1ExtentManager12_getOpenFileEi"},{"b":"400000","o":"995246","s":"_ZNK5mongo19MmapV1ExtentManager12_recordForV1E
|
RKNS_7DiskLocE"},{"b":"400000","o":"99527F","s":"_ZNK5mongo19MmapV1ExtentManager11recordForV1ERKNS_7DiskLocE"},{"b":"400000","o":"9A6516","s":"_ZN5mongo19SimpleRecordStoreV125_allocFro
|
mExistingExtentsEPNS_16OperationContextEi"},{"b":"400000","o":"9A7523","s":"_ZN5mongo19SimpleRecordStoreV111allocRecordEPNS_16OperationContextEib"},{"b":"400000","o":"99C12B","s":"_ZN5
|
mongo17RecordStoreV1Base13_insertRecordEPNS_16OperationContextEPKcib"},{"b":"400000","o":"99C2D9","s":"_ZN5mongo17RecordStoreV1Base12insertRecordEPNS_16OperationContextEPKcib"},{"b":"4
|
00000","o":"5303F1","s":"_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb"},{"b":"400000","o":"530F9C","s":"_ZN5mongo10Collection14insertDocumentEPNS_16Ope
|
rationContextERKNS_7BSONObjEb"},{"b":"400000","o":"66435C","s":"_ZN5mongo11UpdateStage8doInsertEv"},{"b":"400000","o":"665094","s":"_ZN5mongo11UpdateStage4workEPm"},{"b":"400000","o":"
|
80DD32","s":"_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE"},{"b":"400000","o":"80E04B","s":"_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEP
|
NS_8RecordIdE"},{"b":"400000","o":"80E72D","s":"_ZN5mongo12PlanExecutor11executePlanEv"},{"b":"400000","o":"5D7499","s":"_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEP
|
NS_7BSONObjEPPNS_16WriteErrorDetailE"},{"b":"400000","o":"5D8CF5","s":"_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS
|
_19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaISE_EE"},{"b":"400000","o":"5D9325","s":"_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22Bat
|
chedCommandResponseE"},{"b":"400000","o":"5DC00C","s":"_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb"},{"b":"400000","o":"5FED91","s":"_ZN5m
|
ongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb"},{"b":"400000","o":"5FFD9C","s":"_ZN5mongo7Command11execCommandEPNS_16OperationConte
|
xtEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb"},{"b":"400000","o":"600A6B","s":"_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorE
|
EERNS_14BSONObjBuilderEbi"},{"b":"400000","o":"7DB12B","s":"_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_"},{"b":"4000
|
00","o":"6E9AAA","s":"_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE"},{"b":"400000","o":"423335","s":"_ZN5mongo16MyMessageHandler
|
7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE"},{"b":"400000","o":"B72189","s":"_ZN5mongo17PortMessageServer17handleIncomingMsgEPv"},{"b":"3766600000","o":"7AA1"},{
|
"b":"3766200000","o":"E8AAD","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.0.14", "gitVersion" : "08352afcca24bfc145240a0fac9d28b978ab77f3", "uname" : { "sysname" : "Linux", "rel
|
ease" : "2.6.32-642.6.2.el6.x86_64", "version" : "#1 SMP Wed Oct 26 06:52:09 UTC 2016", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "C7C98E2BEC3ED8
|
663669759EECCD26477D4AC85B" }, { "b" : "7FFCB8EFD000", "elfType" : 3, "buildId" : "F19AA6D32E36D9FB2A754E6499A6C4F568C419FB" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "bui
|
ldId" : "B3BD4C596D72FCBE4607C86FEEC14F47B46D0DCC" }, { "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "7CE5058BE3F2AE6EBC0DF61E790C3FCED1A34C7B" }, { "path" : "/usr/li
|
b64/libcrypto.so.10", "elfType" : 3, "buildId" : "096F60DC43484ECA364206125CED7D563FF9D027" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "86FE5BC1F46B8F8AA9A7A479FF991
|
900DB93F720" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "045D39F19533291EBD72D0EE0247F9D49BE2521E" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "B60E
|
F3FCE5D4D2D8BAD2585D5CAAA1167B35DBFD" }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "9A6E4BDFA184364D81F7DFD789474C3FB8F98A00" }, { "path" : "/lib64/libc.so.6", "elf
|
Type" : 3, "buildId" : "F14F3F5B55FC325FDFA7FB6F22F73CA1C6977379" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "DB04BDF55FE66BADD1081D833C17B4881E0EEDB9" },
|
{ "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "9B852585C66329AA02EFB28497E652A40F538E78" }, { "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "EF3AACAF
|
D6BF71BB861F194C1559153FB0B020E2" }, { "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "152E2C18A7A2145021A8A879A01A82EE134E3946" }, { "path" : "/lib64/libk5crypto.so.3",
|
"elfType" : 3, "buildId" : "DDE6774979156442185836150FC0785170F8001F" }, { "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "pat
|
h" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A23DAFBCE170763BF1E836A8B26113F9CD20E0DA" }, { "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7
|
F5B139A0B431D3F06D82014AFF3251" }, { "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "DDF6449707FD4C49DDE32A293EEE9AC218BFC460" }, { "path" : "/lib64/libselinux.so.1", "el
|
fType" : 3, "buildId" : "B4576BE308DDCF7BC31F7304E4734C3D846D0236" } ] }}
|
mongod(_ZN5mongo15printStackTraceERSo+0x32) [0xfc2852]
|
mongod(_ZN5mongo10logContextEPKc+0xE9) [0xf5d8b9]
|
mongod(_ZN5mongo15invariantFailedEPKcS1_j+0xB9) [0xf40c89]
|
mongod(_ZNK5mongo19MmapV1ExtentManager12_getOpenFileEi+0x8F) [0xd9509f]
|
mongod(_ZNK5mongo19MmapV1ExtentManager12_recordForV1ERKNS_7DiskLocE+0x16) [0xd95246]
|
mongod(_ZNK5mongo19MmapV1ExtentManager11recordForV1ERKNS_7DiskLocE+0xF) [0xd9527f]
|
mongod(_ZN5mongo19SimpleRecordStoreV125_allocFromExistingExtentsEPNS_16OperationContextEi+0xE6) [0xda6516]
|
mongod(_ZN5mongo19SimpleRecordStoreV111allocRecordEPNS_16OperationContextEib+0x163) [0xda7523]
|
mongod(_ZN5mongo17RecordStoreV1Base13_insertRecordEPNS_16OperationContextEPKcib+0x5B) [0xd9c12b]
|
mongod(_ZN5mongo17RecordStoreV1Base12insertRecordEPNS_16OperationContextEPKcib+0x79) [0xd9c2d9]
|
mongod(_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x61) [0x9303f1]
|
mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x8C) [0x930f9c]
|
mongod(_ZN5mongo11UpdateStage8doInsertEv+0x17C) [0xa6435c]
|
mongod(_ZN5mongo11UpdateStage4workEPm+0xD4) [0xa65094]
|
mongod(_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0x142) [0xc0dd32]
|
mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x3B) [0xc0e04b]
|
mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x3D) [0xc0e72d]
|
mongod(_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEPNS_7BSONObjEPPNS_16WriteErrorDetailE+0x719) [0x9d7499]
|
mongod(_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaIS
|
E_EE+0x245) [0x9d8cf5]
|
mongod(_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x465) [0x9d9325]
|
mongod(_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x17C) [0x9dc00c]
|
mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x31) [0x9fed91]
|
mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC6C) [0x9ffd9c]
|
mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0xa00a6b]
|
mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x73B) [0xbdb12b]
|
mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xABA) [0xae9aaa]
|
mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xF5) [0x823335]
|
mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x339) [0xf72189]
|
libpthread.so.0(+0x7AA1) [0x3766607aa1]
|
libc.so.6(clone+0x6D) [0x37662e8aad]
|
----- END BACKTRACE -----
|
2016-11-17T13:09:43.865+0000 I - [conn27]
|
|
***aborting after invariant() failure
|
If you need any other information to debug this please let me know.