• Type: Icon: Bug Bug
    • Resolution: Done
    • Priority: Icon: Critical - P2 Critical - P2
    • None
    • Affects Version/s: None
    • Component/s: MMAPv1
    • Labels:
      None
    • ALL

      Hi,

      I have a 3.0.14 mmapv1 server repeatedly crashing with the following stack trace

      w: 1 } }, Collection: { acquireCount: { W: 1 } } } 109ms
      2016-11-17T13:09:43.840+0000 I STORAGE  [conn27] _getOpenFile() invalid file index requested 1832673968
      2016-11-17T13:09:43.840+0000 I -        [conn27] Invariant failure false src/mongo/db/storage/mmap_v1/mmap_v1_extent_manager.cpp 201
      2016-11-17T13:09:43.865+0000 I CONTROL  [conn27]
       0xfc2852 0xf5d8b9 0xf40c89 0xd9509f 0xd95246 0xd9527f 0xda6516 0xda7523 0xd9c12b 0xd9c2d9 0x9303f1 0x930f9c 0xa6435c 0xa65094 0xc0dd32 0xc0e04b 0xc0e72d 0x9d7499 0x9d8cf5 0x9d9325 0x9
      dc00c 0x9fed91 0x9ffd9c 0xa00a6b 0xbdb12b 0xae9aaa 0x823335 0xf72189 0x3766607aa1 0x37662e8aad
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"BC2852","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"B5D8B9","s":"_ZN5mongo10logContextEPKc"},{"b":"400000","o":"B40C89","s":"_ZN5mongo15in
      variantFailedEPKcS1_j"},{"b":"400000","o":"99509F","s":"_ZNK5mongo19MmapV1ExtentManager12_getOpenFileEi"},{"b":"400000","o":"995246","s":"_ZNK5mongo19MmapV1ExtentManager12_recordForV1E
      RKNS_7DiskLocE"},{"b":"400000","o":"99527F","s":"_ZNK5mongo19MmapV1ExtentManager11recordForV1ERKNS_7DiskLocE"},{"b":"400000","o":"9A6516","s":"_ZN5mongo19SimpleRecordStoreV125_allocFro
      mExistingExtentsEPNS_16OperationContextEi"},{"b":"400000","o":"9A7523","s":"_ZN5mongo19SimpleRecordStoreV111allocRecordEPNS_16OperationContextEib"},{"b":"400000","o":"99C12B","s":"_ZN5
      mongo17RecordStoreV1Base13_insertRecordEPNS_16OperationContextEPKcib"},{"b":"400000","o":"99C2D9","s":"_ZN5mongo17RecordStoreV1Base12insertRecordEPNS_16OperationContextEPKcib"},{"b":"4
      00000","o":"5303F1","s":"_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb"},{"b":"400000","o":"530F9C","s":"_ZN5mongo10Collection14insertDocumentEPNS_16Ope
      rationContextERKNS_7BSONObjEb"},{"b":"400000","o":"66435C","s":"_ZN5mongo11UpdateStage8doInsertEv"},{"b":"400000","o":"665094","s":"_ZN5mongo11UpdateStage4workEPm"},{"b":"400000","o":"
      80DD32","s":"_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE"},{"b":"400000","o":"80E04B","s":"_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEP
      NS_8RecordIdE"},{"b":"400000","o":"80E72D","s":"_ZN5mongo12PlanExecutor11executePlanEv"},{"b":"400000","o":"5D7499","s":"_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEP
      NS_7BSONObjEPPNS_16WriteErrorDetailE"},{"b":"400000","o":"5D8CF5","s":"_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS
      _19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaISE_EE"},{"b":"400000","o":"5D9325","s":"_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22Bat
      chedCommandResponseE"},{"b":"400000","o":"5DC00C","s":"_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb"},{"b":"400000","o":"5FED91","s":"_ZN5m
      ongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb"},{"b":"400000","o":"5FFD9C","s":"_ZN5mongo7Command11execCommandEPNS_16OperationConte
      xtEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb"},{"b":"400000","o":"600A6B","s":"_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorE
      EERNS_14BSONObjBuilderEbi"},{"b":"400000","o":"7DB12B","s":"_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_"},{"b":"4000
      00","o":"6E9AAA","s":"_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE"},{"b":"400000","o":"423335","s":"_ZN5mongo16MyMessageHandler
      7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE"},{"b":"400000","o":"B72189","s":"_ZN5mongo17PortMessageServer17handleIncomingMsgEPv"},{"b":"3766600000","o":"7AA1"},{
      "b":"3766200000","o":"E8AAD","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.0.14", "gitVersion" : "08352afcca24bfc145240a0fac9d28b978ab77f3", "uname" : { "sysname" : "Linux", "rel
      ease" : "2.6.32-642.6.2.el6.x86_64", "version" : "#1 SMP Wed Oct 26 06:52:09 UTC 2016", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "C7C98E2BEC3ED8
      663669759EECCD26477D4AC85B" }, { "b" : "7FFCB8EFD000", "elfType" : 3, "buildId" : "F19AA6D32E36D9FB2A754E6499A6C4F568C419FB" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "bui
      ldId" : "B3BD4C596D72FCBE4607C86FEEC14F47B46D0DCC" }, { "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "7CE5058BE3F2AE6EBC0DF61E790C3FCED1A34C7B" }, { "path" : "/usr/li
      b64/libcrypto.so.10", "elfType" : 3, "buildId" : "096F60DC43484ECA364206125CED7D563FF9D027" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "86FE5BC1F46B8F8AA9A7A479FF991
      900DB93F720" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "045D39F19533291EBD72D0EE0247F9D49BE2521E" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "B60E
      F3FCE5D4D2D8BAD2585D5CAAA1167B35DBFD" }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "9A6E4BDFA184364D81F7DFD789474C3FB8F98A00" }, { "path" : "/lib64/libc.so.6", "elf
      Type" : 3, "buildId" : "F14F3F5B55FC325FDFA7FB6F22F73CA1C6977379" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "DB04BDF55FE66BADD1081D833C17B4881E0EEDB9" },
      { "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "9B852585C66329AA02EFB28497E652A40F538E78" }, { "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "EF3AACAF
      D6BF71BB861F194C1559153FB0B020E2" }, { "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "152E2C18A7A2145021A8A879A01A82EE134E3946" }, { "path" : "/lib64/libk5crypto.so.3",
       "elfType" : 3, "buildId" : "DDE6774979156442185836150FC0785170F8001F" }, { "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "pat
      h" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A23DAFBCE170763BF1E836A8B26113F9CD20E0DA" }, { "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7
      F5B139A0B431D3F06D82014AFF3251" }, { "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "DDF6449707FD4C49DDE32A293EEE9AC218BFC460" }, { "path" : "/lib64/libselinux.so.1", "el
      fType" : 3, "buildId" : "B4576BE308DDCF7BC31F7304E4734C3D846D0236" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x32) [0xfc2852]
       mongod(_ZN5mongo10logContextEPKc+0xE9) [0xf5d8b9]
       mongod(_ZN5mongo15invariantFailedEPKcS1_j+0xB9) [0xf40c89]
       mongod(_ZNK5mongo19MmapV1ExtentManager12_getOpenFileEi+0x8F) [0xd9509f]
       mongod(_ZNK5mongo19MmapV1ExtentManager12_recordForV1ERKNS_7DiskLocE+0x16) [0xd95246]
       mongod(_ZNK5mongo19MmapV1ExtentManager11recordForV1ERKNS_7DiskLocE+0xF) [0xd9527f]
       mongod(_ZN5mongo19SimpleRecordStoreV125_allocFromExistingExtentsEPNS_16OperationContextEi+0xE6) [0xda6516]
       mongod(_ZN5mongo19SimpleRecordStoreV111allocRecordEPNS_16OperationContextEib+0x163) [0xda7523]
       mongod(_ZN5mongo17RecordStoreV1Base13_insertRecordEPNS_16OperationContextEPKcib+0x5B) [0xd9c12b]
       mongod(_ZN5mongo17RecordStoreV1Base12insertRecordEPNS_16OperationContextEPKcib+0x79) [0xd9c2d9]
       mongod(_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x61) [0x9303f1]
       mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x8C) [0x930f9c]
       mongod(_ZN5mongo11UpdateStage8doInsertEv+0x17C) [0xa6435c]
       mongod(_ZN5mongo11UpdateStage4workEPm+0xD4) [0xa65094]
       mongod(_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0x142) [0xc0dd32]
       mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x3B) [0xc0e04b]
       mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x3D) [0xc0e72d]
       mongod(_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEPNS_7BSONObjEPPNS_16WriteErrorDetailE+0x719) [0x9d7499]
       mongod(_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaIS
      E_EE+0x245) [0x9d8cf5]
       mongod(_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x465) [0x9d9325]
       mongod(_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x17C) [0x9dc00c]
       mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x31) [0x9fed91]
       mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC6C) [0x9ffd9c]
       mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0xa00a6b]
       mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x73B) [0xbdb12b]
       mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xABA) [0xae9aaa]
       mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xF5) [0x823335]
       mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x339) [0xf72189]
       libpthread.so.0(+0x7AA1) [0x3766607aa1]
       libc.so.6(clone+0x6D) [0x37662e8aad]
      -----  END BACKTRACE  -----
      2016-11-17T13:09:43.865+0000 I -        [conn27]
      
      ***aborting after invariant() failure
      

      If you need any other information to debug this please let me know.

            Assignee:
            kelsey.schubert@mongodb.com Kelsey Schubert
            Reporter:
            dharshanr@scalegrid.net Dharshan Rangegowda
            Votes:
            0 Vote for this issue
            Watchers:
            5 Start watching this issue

              Created:
              Updated:
              Resolved: