Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-24408

Server crashed - Cannot allocate memory

    • Type: Icon: Bug Bug
    • Resolution: Done
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: 3.2.6
    • Component/s: WiredTiger
    • None
    • ALL
    • None
    • 3
    • None
    • None
    • None
    • None
    • None
    • None

      I'm running MongoDB 3.2.6 on an EC2 instance, and the server crashed with the following error:

      2016-06-06T05:27:46.527+0000 E STORAGE  [conn326] WiredTiger (12) [1465190866:521095][1721:0x7fdbf460a700], file:collection-2--1597448640633402554.wt, WT_CURSOR.next: memory allocation of 3379628 bytes failed: Cannot allocate memory
      2016-06-06T05:27:46.534+0000 I -        [conn326] Invariant failure: advanceRet resulted in status UnknownError: 12: Cannot allocate memory at src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp 481
      2016-06-06T05:27:46.587+0000 I CONTROL  [conn326]
       0x1315982 0x12b3798 0x12a067d 0x109084f 0xbe80d0 0xbe9800 0xe2b435 0xe2baf9 0xe2bbf5 0xb3484b 0xbc5f03 0xbc6d74 0xb22dc0 0xcd4e25 0x9b78cc 0x12c2e7d 0x7fdc11c35182 0x7fdc1196247d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"F15982","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"EB3798","s":"_ZN5mongo10logContextEPKc"},{"b":"400000","o":"EA067D","s":"_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j"},{"b":"400000","o":"C9084F","s":"_ZN5mongo21WiredTigerRecordStore6Cursor4nextEv"},{"b":"400000","o":"7E80D0","s":"_ZN5mongo14CollectionScan4workEPm"},{"b":"400000","o":"7E9800","s":"_ZN5mongo10CountStage4workEPm"},{"b":"400000","o":"A2B435","s":"_ZN5mongo12PlanExecutor11getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE"},{"b":"400000","o":"A2BAF9","s":"_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE"},{"b":"400000","o":"A2BBF5","s":"_ZN5mongo12PlanExecutor11executePlanEv"},{"b":"400000","o":"73484B"},{"b":"400000","o":"7C5F03","s":"_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE"},{"b":"400000","o":"7C6D74","s":"_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE"},{"b":"400000","o":"722DC0","s":"_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE"},{"b":"400000","o":"8D4E25","s":"_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE"},{"b":"400000","o":"5B78CC","s":"_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortE"},{"b":"400000","o":"EC2E7D","s":"_ZN5mongo17PortMessageServer17handleIncomingMsgEPv"},{"b":"7FDC11C2D000","o":"8182"},{"b":"7FDC11868000","o":"FA47D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.2.6", "gitVersion" : "05552b562c7a0b3143a729aaa0838e558dc49b25", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.13.0-74-generic", "version" : "#118-Ubuntu SMP Thu Dec 17 22:52:10 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "B35B0D2B9CB934315BA086AB5C88C919B9B5BA50" }, { "b" : "7FFFFD815000", "elfType" : 3, "buildId" : "DC075B751E9FB361F14CD59BD81300A6BB5CB377" }, { "b" : "7FDC12B4F000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "74864DB9D5F69D39A67E4755012FB6573C469B3D" }, { "b" : "7FDC12773000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "AAE7CFF8351B730830BDBCE0DCABBE06574B7144" }, { "b" : "7FDC1256B000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "B376100CAB1EAC4E5DE066EACFC282BF7C0B54F3" }, { "b" : "7FDC12367000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "67699FFDA9FD2A552032E0652A242E82D65AA10D" }, { "b" : "7FDC12061000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "EF3F6DFFA1FBE48436EC6F45CD3AABA157064BB4" }, { "b" : "7FDC11E4B000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "36311B4457710AE5578C4BF00791DED7359DBB92" }, { "b" : "7FDC11C2D000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "AF06068681750736E0524DF17D5A86CB2C3F765C" }, { "b" : "7FDC11868000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "5382058B69031CAA9B9996C11061CD164C9398FF" }, { "b" : "7FDC12DAE000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "2A816C3EBBA4E12813FBD34B06FBD25BC892A67F" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x1315982]
       mongod(_ZN5mongo10logContextEPKc+0x138) [0x12b3798]
       mongod(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0xAD) [0x12a067d]
       mongod(_ZN5mongo21WiredTigerRecordStore6Cursor4nextEv+0x45F) [0x109084f]
       mongod(_ZN5mongo14CollectionScan4workEPm+0x940) [0xbe80d0]
       mongod(_ZN5mongo10CountStage4workEPm+0xD0) [0xbe9800]
       mongod(_ZN5mongo12PlanExecutor11getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0x275) [0xe2b435]
       mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x39) [0xe2baf9]
       mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x55) [0xe2bbf5]
       mongod(+0x73484B) [0xb3484b]
       mongod(_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE+0x463) [0xbc5f03]
       mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE+0x404) [0xbc6d74]
       mongod(_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE+0x1F0) [0xb22dc0]
       mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB45) [0xcd4e25]
       mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortE+0xEC) [0x9b78cc]
       mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x26D) [0x12c2e7d]
       libpthread.so.0(+0x8182) [0x7fdc11c35182]
       libc.so.6(clone+0x6D) [0x7fdc1196247d]
      -----  END BACKTRACE  -----
      

            Assignee:
            Unassigned Unassigned
            Reporter:
            omers Omer Segev
            Votes:
            0 Vote for this issue
            Watchers:
            5 Start watching this issue

              Created:
              Updated:
              Resolved: