Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-16623

Cannot allocate memory (wiredtiger) (mms-dev)

    • Fully Compatible
    • ALL

      I recently upgraded the mms-dev primary from a 2.8.0rc3-pre (e067fff4e3f3079d070ec168f32c24db9a51a944) to 2.8.0-rc3. After about a half an hour, the primary had a segfault with the following message.

      This node has not had a fresh sync on 2.8.0-rc3. I believe the last time it had a fresh sync was 2.8.0-rc2.

      2014-12-21T20:29:44.685+0000 E STORAGE  [conn1337] WiredTiger (12) [1419193784:682102][8148:0x7f0d3ebdd700], file:collection-8--5314993906140581602.wt, cursor.insert: memory allocation: Cannot allocate memory
      2014-12-21T20:29:44.718+0000 F -        [conn1337] Invalid access at address: 0x30b18
      2014-12-21T20:29:44.783+0000 F -        [conn1337] Got signal: 11 (Segmentation fault).
      
       0xf0bd99 0xf0b442 0xf0b76e 0x7f0d7e286340 0x13141b8 0x1318967 0x1319f7a 0x12e99a7 0x12ea2bd 0x12e7e6d 0x129fa9e 0x12d5d06 0xd33918 0xd30c0d 0x8ebb5c 0xc12e8f 0xc109c3 0xa08df1 0xa099cc 0xba969a 0xbaa42d 0x989c70 0x98aec2 0x98b405 0x98d4bd 0x9a9784 0x9aa5d3 0x9ab08b 0xb77c2a 0xa8af55 0x7e1770 0xec9d61 0x7f0d7e27e182 0x7f0d7d37efbd
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"B0BD99"},{"b":"400000","o":"B0B442"},{"b":"400000","o":"B0B76E"},{"b":"7F0D7E276000","o":"10340"},{"b":"400000","o":"F141B8"},{"b":"400000","o":"F18967"},{"b":"400000","o":"F19F7A"},{"b":"400000","o":"EE99A7"},{"b":"400000","o":"EEA2BD"},{"b":"400000","o":"EE7E6D"},{"b":"400000","o":"E9FA9E"},{"b":"400000","o":"ED5D06"},{"b":"400000","o":"933918"},{"b":"400000","o":"930C0D"},{"b":"400000","o":"4EBB5C"},{"b":"400000","o":"812E8F"},{"b":"400000","o":"8109C3"},{"b":"400000","o":"608DF1"},{"b":"400000","o":"6099CC"},{"b":"400000","o":"7A969A"},{"b":"400000","o":"7AA42D"},{"b":"400000","o":"589C70"},{"b":"400000","o":"58AEC2"},{"b":"400000","o":"58B405"},{"b":"400000","o":"58D4BD"},{"b":"400000","o":"5A9784"},{"b":"400000","o":"5AA5D3"},{"b":"400000","o":"5AB08B"},{"b":"400000","o":"777C2A"},{"b":"400000","o":"68AF55"},{"b":"400000","o":"3E1770"},{"b":"400000","o":"AC9D61"},{"b":"7F0D7E276000","o":"8182"},{"b":"7F0D7D284000","o":"FAFBD"}],"processInfo":{ "mongodbVersion" : "2.8.0-rc3", "gitVersion" : "2d679247f17dab05a492c8b6d2c250dab18e54f2", "uname" : { "sysname" : "Linux", "release" : "3.13.0-36-generic", "version" : "#63-Ubuntu SMP Wed Sep 3 21:30:07 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000" }, { "b" : "7FFF82EFE000", "elfType" : 3 }, { "b" : "7F0D7E276000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3 }, { "b" : "7F0D7E06E000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3 }, { "b" : "7F0D7DE6A000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3 }, { "b" : "7F0D7DB66000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3 }, { "b" : "7F0D7D860000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3 }, { "b" : "7F0D7D64A000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3 }, { "b" : "7F0D7D284000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3 }, { "b" : "7F0D7E494000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3 } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf0bd99]
       mongod(+0xB0B442) [0xf0b442]
       mongod(+0xB0B76E) [0xf0b76e]
       libpthread.so.0(+0x10340) [0x7f0d7e286340]
       mongod(+0xF141B8) [0x13141b8]
       mongod(+0xF18967) [0x1318967]
       mongod(__wt_reconcile+0x1AA) [0x1319f7a]
       mongod(+0xEE99A7) [0x12e99a7]
       mongod(__wt_evict+0x7D) [0x12ea2bd]
       mongod(__wt_evict_page+0x2D) [0x12e7e6d]
       mongod(__wt_btcur_insert+0x12CE) [0x129fa9e]
       mongod(+0xED5D06) [0x12d5d06]
       mongod(_ZN5mongo21WiredTigerRecordStore12insertRecordEPNS_16OperationContextEPKcib+0xC8) [0xd33918]
       mongod(_ZN5mongo21WiredTigerRecordStore12insertRecordEPNS_16OperationContextEPKNS_9DocWriterEb+0x8D) [0xd30c0d]
       mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextEPKNS_9DocWriterEb+0x5C) [0x8ebb5c]
       mongod(+0x812E8F) [0xc12e8f]
       mongod(_ZN5mongo4repl5logOpEPNS_16OperationContextEPKcS4_RKNS_7BSONObjEPS5_Pbb+0xA3) [0xc109c3]
       mongod(_ZN5mongo11UpdateStage8doInsertEv+0x511) [0xa08df1]
       mongod(_ZN5mongo11UpdateStage4workEPm+0xCC) [0xa099cc]
       mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x9A) [0xba969a]
       mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x3D) [0xbaa42d]
       mongod(_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEPNS_7BSONObjEPPNS_16WriteErrorDetailE+0x610) [0x989c70]
       mongod(_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestEPSt6vectorIPNS_19BatchedUpsertDetailESaIS6_EEPS4_IPNS_16WriteErrorDetailESaISB_EE+0x1F2) [0x98aec2]
       mongod(_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x395) [0x98b405]
       mongod(_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x15D) [0x98d4bd]
       mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9a9784]
       mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC13) [0x9aa5d3]
       mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9ab08b]
       mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERNS_5CurOpES3_b+0x76A) [0xb77c2a]
       mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortEb+0xB25) [0xa8af55]
       mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xE0) [0x7e1770]
       mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x411) [0xec9d61]
       libpthread.so.0(+0x8182) [0x7f0d7e27e182]
       libc.so.6(clone+0x6D) [0x7f0d7d37efbd]
      -----  END BACKTRACE  -----
      

        1. osx.png
          osx.png
          23 kB
        2. mmsdev-2015-01-02.png
          mmsdev-2015-01-02.png
          201 kB
        3. mmsdev-2015-01-01.png
          mmsdev-2015-01-01.png
          374 kB
        4. linux.png
          linux.png
          19 kB
        5. incident1-oplog.png
          incident1-oplog.png
          143 kB
        6. incident1.png
          incident1.png
          533 kB
        7. 54972d6de4b0c8d9d7bcaca1.json
          112 kB
        8. 12-21-mem.png
          12-21-mem.png
          138 kB

            Assignee:
            michael.cahill@mongodb.com Michael Cahill (Inactive)
            Reporter:
            cailin.nelson@mongodb.com Cailin Nelson (Inactive)
            Votes:
            1 Vote for this issue
            Watchers:
            12 Start watching this issue

              Created:
              Updated:
              Resolved: