Mongo nodes dying with Segmentation fault

XMLWordPrintableJSON

    • Type: Bug
    • Resolution: Done
    • Priority: Major - P3
    • None
    • Affects Version/s: 3.4.15
    • Component/s: None
    • None
    • Fully Compatible
    • None
    • 3
    • None
    • None
    • None
    • None
    • None
    • None

      We have seen this happen a couple of times now.  It would be great to find the root cause of the problem.

      2020-01-25T05:10:21.131+0000 I NETWORK [conn70233004] received client metadata from 127.0.0.1:55388 conn70233004: { application: { name: "MongoDB Shell" }, driver: { name: "MongoDB Internal Client", version: "3.4.15" }, os: { type: "Linux", name: "Amazon Linux AMI release 2018.03", architecture: "x86_64", version: "Kernel 4.14.47-56.37.amzn1.x86_64" } }
      2020-01-25T05:10:21.169+0000 F - [conn70215932] Got signal: 11 (Segmentation fault).
      
      0x55e0dfa43371 0x55e0dfa42589 0x55e0dfa42bf6 0x7f6cf5387690 0x55e0e0374b40 0x55e0e0395a90 0x55e0e0363ee2 0x55e0e03b2b14 0x55e0df73c6a8 0x55e0df73cbb5 0x55e0deea7d9f 0x55e0df446512 0x55e0df449307 0x55e0df1a5238 0x55e0deeac3ea 0x55e0deeac673 0x55e0df0a18b0 0x55e0df0a3044 0x55e0df06f333 0x55e0df37962a 0x55e0df379f4b 0x55e0df37a07d 0x55e0df1d8da6 0x55e0deffa9ff 0x55e0deff637c 0x55e0def31cb5 0x55e0def32fa1 0x55e0df550e00 0x55e0df150f12 0x55e0df152f16 0x55e0ded5276d 0x55e0ded5309d 0x55e0df9aa251 0x7f6cf537fde5 0x7f6cf50a9bad
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"55E0DE4C6000","o":"157D371","s":"_ZN5mongo15printStackTraceERSo"},{"b":"55E0DE4C6000","o":"157C589"},{"b":"55E0DE4C6000","o":"157CBF6"},{"b":"7F6CF5378000","o":"F690"},{"b":"55E0DE4C6000","o":"1EAEB40","s":"__wt_page_in_func"},{"b":"55E0DE4C6000","o":"1ECFA90","s":"__wt_row_search"},{"b":"55E0DE4C6000","o":"1E9DEE2","s":"__wt_btcur_insert"},{"b":"55E0DE4C6000","o":"1EECB14"},{"b":"55E0DE4C6000","o":"12766A8","s":"_ZN5mongo21WiredTigerRecordStore14_insertRecordsEPNS_16OperationContextEPNS_6RecordEm"},{"b":"55E0DE4C6000","o":"1276BB5","s":"_ZN5mongo21WiredTigerRecordStore26insertRecordsWithDocWriterEPNS_16OperationContextEPKPKNS_9DocWriterEmPNS_8RecordIdE"},{"b":"55E0DE4C6000","o":"9E1D9F","s":"_ZN5mongo10Collection23insertDocumentsForOplogEPNS_16OperationContextEPKPKNS_9DocWriterEm"},{"b":"55E0DE4C6000","o":"F80512","s":"_ZN5mongo4repl12_logOpsInnerEPNS_16OperationContextERKNS_15NamespaceStringEPKPKNS_9DocWriterEmPNS_10CollectionENS0_22ReplicationCoordinator4ModeENS0_6OpTimeE"},{"b":"55E0DE4C6000","o":"F83307","s":"_ZN5mongo4repl6logOpsEPNS_16OperationContextEPKcRKNS_15NamespaceStringEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorISA_SaISA_EEEESG_b"},{"b":"55E0DE4C6000","o":"CDF238","s":"_ZN5mongo14OpObserverImpl9onInsertsEPNS_16OperationContextERKNS_15NamespaceStringEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorIS8_SaIS8_EEEESE_b"},{"b":"55E0DE4C6000","o":"9E63EA","s":"_ZN5mongo10Collection15insertDocumentsEPNS_16OperationContextEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorIS5_SaIS5_EEEESB_PNS_7OpDebugEbb"},{"b":"55E0DE4C6000","o":"9E6673","s":"_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEPNS_7OpDebugEbb"},{"b":"55E0DE4C6000","o":"BDB8B0","s":"_ZN5mongo11UpdateStage8doInsertEv"},{"b":"55E0DE4C6000","o":"BDD044","s":"_ZN5mongo11UpdateStage6doWorkEPm"},{"b":"55E0DE4C6000","o":"BA9333","s":"_ZN5mongo9PlanStage4workEPm"},{"b":"55E0DE4C6000","o":"EB362A","s":"_ZN5mongo12PlanExecutor11getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE"},{"b":"55E0DE4C6000","o":"EB3F4B","s":"_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE"},{"b":"55E0DE4C6000","o":"EB407D","s":"_ZN5mongo12PlanExecutor11executePlanEv"},{"b":"55E0DE4C6000","o":"D12DA6","s":"_ZN5mongo14performUpdatesEPNS_16OperationContextERKNS_8UpdateOpE"},{"b":"55E0DE4C6000","o":"B349FF","s":"_ZN5mongo9CmdUpdate7runImplEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERNS_14BSONObjBuilderE"},{"b":"55E0DE4C6000","o":"B3037C"},{"b":"55E0DE4C6000","o":"A6BCB5","s":"_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE"},{"b":"55E0DE4C6000","o":"A6CFA1","s":"_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE"},{"b":"55E0DE4C6000","o":"108AE00","s":"_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE"},{"b":"55E0DE4C6000","o":"C8AF12"},{"b":"55E0DE4C6000","o":"C8CF16","s":"_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE"},{"b":"55E0DE4C6000","o":"88C76D","s":"_ZN5mongo23ServiceEntryPointMongod12_sessionLoopERKSt10shared_ptrINS_9transport7SessionEE"},{"b":"55E0DE4C6000","o":"88D09D"},{"b":"55E0DE4C6000","o":"14E4251"},{"b":"7F6CF5378000","o":"7DE5"},{"b":"7F6CF4FAB000","o":"FEBAD","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.4.15", "gitVersion" : "52e5b5fbaa3a2a5b1a217f5e647b5061817475f9", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "4.14.47-56.37.amzn1.x86_64", "version" : "#1 SMP Wed Jun 6 18:49:01 UTC 2018", "machine" : "x86_64" }, "somap" : [ { "b" : "55E0DE4C6000", "elfType" : 3, "buildId" : "735C6F8FB5872DFC52902334A4BBB6D423200D89" }, { "b" : "7FFDC98DF000", "elfType" : 3, "buildId" : "2DF3D53B81C4CFBB2F14430578A041B78D5A1EE2" }, { "b" : "7F6CF6316000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "9C4EB34A346260F2A77746F4E5ED837619137DB7" }, { "b" : "7F6CF5EB8000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "EC480B38432587A9B21BFBD917EF020731EBD2CF" }, { "b" : "7F6CF5CB0000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "F2701E2A24459D5B55DF5549D585F091E7BCF07A" }, { "b" : "7F6CF5AAC000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "0E5CD5BAA5EE8BF3648A5031B088F9A78C89364F" }, { "b" : "7F6CF57AA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "07FB92AFEF1756F093371CE60C3AE85DD3A06325" }, { "b" : "7F6CF5594000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "A03C9A80E995ED5F43077AB754A258FA0E34C3CD" }, { "b" : "7F6CF5378000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "D973C39D1900DC61D8519C653C3BC405692DE563" }, { "b" : "7F6CF4FAB000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "AF310F56618FC1EF9158973484F60942F11CC0FB" }, { "b" : "7F6CF6587000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "8402047FD4A85B3CD1142346EA06BCD6E15A82A3" }, { "b" : "7F6CF4D5E000", "path" : "/usr/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "6FBBD34B86296FDF883FE5122017EC5CD3F98ED7" }, { "b" : "7F6CF4A76000", "path" : "/usr/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "76429E6FD408BBB675798D6458F2735383710D0B" }, { "b" : "7F6CF4873000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7F6CF4640000", "path" : "/usr/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "5B2A76F1EF91EDAA0494BE680CADAFE6489326E1" }, { "b" : "7F6CF442A000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7F6CF421C000", "path" : "/usr/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "3ACB59488C6D8DE0A1F4F1B0C290A570D9E42F3D" }, { "b" : "7F6CF4019000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7F6CF3E00000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "9E5E0BF5F22DE7555BC4B9853240817147489258" }, { "b" : "7F6CF3BDF000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "F5054DC94443326819FBF3065CFDF5E4726F57EE" } ] }}
      mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x55e0dfa43371]
      mongod(+0x157C589) [0x55e0dfa42589]
      mongod(+0x157CBF6) [0x55e0dfa42bf6]
      libpthread.so.0(+0xF690) [0x7f6cf5387690]
      mongod(__wt_page_in_func+0x1010) [0x55e0e0374b40]
      mongod(__wt_row_search+0x660) [0x55e0e0395a90]
      mongod(__wt_btcur_insert+0xB72) [0x55e0e0363ee2]
      mongod(+0x1EECB14) [0x55e0e03b2b14]
      mongod(_ZN5mongo21WiredTigerRecordStore14_insertRecordsEPNS_16OperationContextEPNS_6RecordEm+0x208) [0x55e0df73c6a8]
      mongod(_ZN5mongo21WiredTigerRecordStore26insertRecordsWithDocWriterEPNS_16OperationContextEPKPKNS_9DocWriterEmPNS_8RecordIdE+0x1B5) [0x55e0df73cbb5]
      mongod(_ZN5mongo10Collection23insertDocumentsForOplogEPNS_16OperationContextEPKPKNS_9DocWriterEm+0x6F) [0x55e0deea7d9f]
      mongod(_ZN5mongo4repl12_logOpsInnerEPNS_16OperationContextERKNS_15NamespaceStringEPKPKNS_9DocWriterEmPNS_10CollectionENS0_22ReplicationCoordinator4ModeENS0_6OpTimeE+0x92) [0x55e0df446512]
      mongod(_ZN5mongo4repl6logOpsEPNS_16OperationContextEPKcRKNS_15NamespaceStringEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorISA_SaISA_EEEESG_b+0x607) [0x55e0df449307]
      mongod(_ZN5mongo14OpObserverImpl9onInsertsEPNS_16OperationContextERKNS_15NamespaceStringEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorIS8_SaIS8_EEEESE_b+0x38) [0x55e0df1a5238]
      mongod(_ZN5mongo10Collection15insertDocumentsEPNS_16OperationContextEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorIS5_SaIS5_EEEESB_PNS_7OpDebugEbb+0x45A) [0x55e0deeac3ea]
      mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEPNS_7OpDebugEbb+0x83) [0x55e0deeac673]
      mongod(_ZN5mongo11UpdateStage8doInsertEv+0x1D0) [0x55e0df0a18b0]
      mongod(_ZN5mongo11UpdateStage6doWorkEPm+0xF4) [0x55e0df0a3044]
      mongod(_ZN5mongo9PlanStage4workEPm+0x63) [0x55e0df06f333]
      mongod(_ZN5mongo12PlanExecutor11getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0x19A) [0x55e0df37962a]
      mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x4B) [0x55e0df379f4b]
      mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x6D) [0x55e0df37a07d]
      mongod(_ZN5mongo14performUpdatesEPNS_16OperationContextERKNS_8UpdateOpE+0x716) [0x55e0df1d8da6]
      mongod(_ZN5mongo9CmdUpdate7runImplEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERNS_14BSONObjBuilderE+0x4F) [0x55e0deffa9ff]
      mongod(+0xB3037C) [0x55e0deff637c]
      mongod(_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE+0x935) [0x55e0def31cb5]
      mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE+0xF81) [0x55e0def32fa1]
      mongod(_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE+0x240) [0x55e0df550e00]
      mongod(+0xC8AF12) [0x55e0df150f12]
      mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0x746) [0x55e0df152f16]
      mongod(_ZN5mongo23ServiceEntryPointMongod12_sessionLoopERKSt10shared_ptrINS_9transport7SessionEE+0x1FD) [0x55e0ded5276d]
      mongod(+0x88D09D) [0x55e0ded5309d]
      mongod(+0x14E4251) [0x55e0df9aa251]
      libpthread.so.0(+0x7DE5) [0x7f6cf537fde5]
      libc.so.6(clone+0x6D) [0x7f6cf50a9bad]
      ----- END BACKTRACE -----
       

            Assignee:
            Dmitry Agranat
            Reporter:
            Michael Eller
            Votes:
            0 Vote for this issue
            Watchers:
            5 Start watching this issue

              Created:
              Updated:
              Resolved: