Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-21380

Mongo Data node frequently going with "out of memory" message

    • Type: Icon: Bug Bug
    • Resolution: Done
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: None
    • Component/s: Admin
    • None
    • ALL

      I've a mongo cluster version: 3.0.6 with 3 config nodes, 2 data nodes and 1 router. The data nodes frequently goes down multiple times a day with the below error. Would I be having data loss because of it?How do I fix it?
      We perform bulk inserts.

      2015-11-06T13:54:39.730+0000 I WRITE    [conn366] update azuga.locationStore query: { locationStagingId: "ea2e3520-848d-11e5-81a1-7f82ee1dc77e" } update: { $set: { add
      ress: "US-41, Oak Creek, WI 53154, USA", city: "Milwaukee County", state: "Wisconsin", country: "US", zip: "53154", updatedAt: new Date(1446818079299) } } nscanned:0 n
      scannedObjects:0 nMatched:0 nModified:0 keyUpdates:0 writeConflicts:0 numYields:0 locks:{ Global: { acquireCount: { r: 1, w: 1 } }, MMAPV1Journal: { acquireCount: { w:
       2 } }, Database: { acquireCount: { w: 1 } }, Collection: { acquireCount: { W: 1 }, acquireWaitCount: { W: 1 }, timeAcquiringMicros: { W: 434317 } } } 434ms
      2015-11-06T13:54:39.794+0000 F -        [conn183] out of memory.
      
       0xf5bfc9 0xf5baf9 0x1405c4f 0x9bc734 0xa0d1f4 0xa00c4d 0xa0e4e6 0xa37acc 0xbce554 0xbce924 0xbcef8d 0x9b07dd 0x9b1fbc 0x9b24fb 0x9b4ead 0x9d6234 0x9d71bd 0x9d7ecb 0xb
      9cac6 0xab2e20 0x80e0ad 0xf0ef0b 0x7f40b676e182 0x7f40b523647d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"B5BFC9"},{"b":"400000","o":"B5BAF9"},{"b":"400000","o":"1005C4F"},{"b":"400000","o":"5BC734"},{"b":"400000","o":"60D1F4"},{"b":"400000
      ","o":"600C4D"},{"b":"400000","o":"60E4E6"},{"b":"400000","o":"637ACC"},{"b":"400000","o":"7CE554"},{"b":"400000","o":"7CE924"},{"b":"400000","o":"7CEF8D"},{"b":"40000
      0","o":"5B07DD"},{"b":"400000","o":"5B1FBC"},{"b":"400000","o":"5B24FB"},{"b":"400000","o":"5B4EAD"},{"b":"400000","o":"5D6234"},{"b":"400000","o":"5D71BD"},{"b":"4000
      00","o":"5D7ECB"},{"b":"400000","o":"79CAC6"},{"b":"400000","o":"6B2E20"},{"b":"400000","o":"40E0AD"},{"b":"400000","o":"B0EF0B"},{"b":"7F40B6766000","o":"8182"},{"b":
      "7F40B513C000","o":"FA47D"}],"processInfo":{ "mongodbVersion" : "3.0.6", "gitVersion" : "1ef45a23a4c5e3480ac919b28afcba3c615488f2", "uname" : { "sysname" : "Linux", "r
      elease" : "3.13.0-48-generic", "version" : "#80-Ubuntu SMP Thu Mar 12 11:16:15 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId
      " : "BF5AC37B50D416FD8D6D427E561426ED60291032" }, { "b" : "7FFF5E0A7000", "elfType" : 3, "buildId" : "341443CC258F9F27709CAA36C8CB321E4BBFFF95" }, { "b" : "7F40B6766000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "9318E8AF0BFBE444731BB0461202EF57F7C39542" }, { "b" : "7F40B6508000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "FF43D0947510134A8A494063A3C1CF3CEBB27791" }, { "b" : "7F40B612D000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "B927879B878D90DD9FF4B15B00E7799AA8E0272F" }, { "b" : "7F40B5F25000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "92FCF41EFE012D6186E31A59AD05BDBB487769AB" }, { "b" : "7F40B5D21000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "C1AE4CB7195D337A77A3C689051DABAA3980CA0C" }, { "b" : "7F40B5A1D000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3, "buildId" : "19EFDDAB11B3BF5C71570078C59F91CF6592CE9E" }, { "b" : "7F40B5717000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "1D76B71E905CB867B27CEF230FCB20F01A3178F5" }, { "b" : "7F40B5501000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "8D0AA71411580EE6C08809695C3984769F25725B" }, { "b" : "7F40B513C000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "30C94DC66A1FE95180C3D68D2B89E576D5AE213C" }, { "b" : "7F40B6984000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "9F00581AB3C73E3AEA35995A0C50D24D59A01D47" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf5bfc9]
      mongod(_ZN5mongo29reportOutOfMemoryErrorAndExitEv+0x49) [0xf5baf9]
       mongod(tc_new+0x1AF) [0x1405c4f]
       mongod(_ZNSt6vectorImSaImEE19_M_emplace_back_auxIIRKmEEEvDpOT_+0x34) [0x9bc734]
       mongod(_ZN5mongo9IndexScan4workEPm+0x2D4) [0xa0d1f4]
       mongod(_ZN5mongo10FetchStage4workEPm+0xCD) [0xa00c4d]
       mongod(_ZN5mongo18KeepMutationsStage4workEPm+0xE6) [0xa0e4e6]
       mongod(_ZN5mongo11UpdateStage4workEPm+0x7C) [0xa37acc]
       mongod(_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0xA4) [0xbce554]
       mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x34) [0xbce924]
       mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x3D) [0xbcef8d]
       mongod(_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEPNS_7BSONObjEPPNS_16WriteErrorDetailE+0x71D) [0x9b07dd]
       mongod(_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16Writ
      eErrorDetailESaISE_EE+0x23C) [0x9b1fbc]
       mongod(_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x37B) [0x9b24fb]
       mongod(_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x15D) [0x9b4ead]
       mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9d6234]
       mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9d71bd]
       mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9d7ecb]
       mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x746) [0xb9cac6]
       mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xab2e20]
       mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x80e0ad]
       mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf0ef0b]
       libpthread.so.0(+0x8182) [0x7f40b676e182]
       libc.so.6(clone+0x6D) [0x7f40b523647d]
      -----  END BACKTRACE  -----
      

            Assignee:
            Unassigned Unassigned
            Reporter:
            debasishn Debasish Nayak
            Votes:
            0 Vote for this issue
            Watchers:
            3 Start watching this issue

              Created:
              Updated:
              Resolved: