Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-45808

Mongo nodes dying with Segmentation fault

    XMLWordPrintableJSON

Details

    • Icon: Bug Bug
    • Resolution: Done
    • Icon: Major - P3 Major - P3
    • None
    • 3.4.15
    • None
    • None
    • Fully Compatible

    Description

      We have seen this happen a couple of times now.  It would be great to find the root cause of the problem.

      2020-01-25T05:10:21.131+0000 I NETWORK [conn70233004] received client metadata from 127.0.0.1:55388 conn70233004: { application: { name: "MongoDB Shell" }, driver: { name: "MongoDB Internal Client", version: "3.4.15" }, os: { type: "Linux", name: "Amazon Linux AMI release 2018.03", architecture: "x86_64", version: "Kernel 4.14.47-56.37.amzn1.x86_64" } }
      2020-01-25T05:10:21.169+0000 F - [conn70215932] Got signal: 11 (Segmentation fault).
       
      0x55e0dfa43371 0x55e0dfa42589 0x55e0dfa42bf6 0x7f6cf5387690 0x55e0e0374b40 0x55e0e0395a90 0x55e0e0363ee2 0x55e0e03b2b14 0x55e0df73c6a8 0x55e0df73cbb5 0x55e0deea7d9f 0x55e0df446512 0x55e0df449307 0x55e0df1a5238 0x55e0deeac3ea 0x55e0deeac673 0x55e0df0a18b0 0x55e0df0a3044 0x55e0df06f333 0x55e0df37962a 0x55e0df379f4b 0x55e0df37a07d 0x55e0df1d8da6 0x55e0deffa9ff 0x55e0deff637c 0x55e0def31cb5 0x55e0def32fa1 0x55e0df550e00 0x55e0df150f12 0x55e0df152f16 0x55e0ded5276d 0x55e0ded5309d 0x55e0df9aa251 0x7f6cf537fde5 0x7f6cf50a9bad
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"55E0DE4C6000","o":"157D371","s":"_ZN5mongo15printStackTraceERSo"},{"b":"55E0DE4C6000","o":"157C589"},{"b":"55E0DE4C6000","o":"157CBF6"},{"b":"7F6CF5378000","o":"F690"},{"b":"55E0DE4C6000","o":"1EAEB40","s":"__wt_page_in_func"},{"b":"55E0DE4C6000","o":"1ECFA90","s":"__wt_row_search"},{"b":"55E0DE4C6000","o":"1E9DEE2","s":"__wt_btcur_insert"},{"b":"55E0DE4C6000","o":"1EECB14"},{"b":"55E0DE4C6000","o":"12766A8","s":"_ZN5mongo21WiredTigerRecordStore14_insertRecordsEPNS_16OperationContextEPNS_6RecordEm"},{"b":"55E0DE4C6000","o":"1276BB5","s":"_ZN5mongo21WiredTigerRecordStore26insertRecordsWithDocWriterEPNS_16OperationContextEPKPKNS_9DocWriterEmPNS_8RecordIdE"},{"b":"55E0DE4C6000","o":"9E1D9F","s":"_ZN5mongo10Collection23insertDocumentsForOplogEPNS_16OperationContextEPKPKNS_9DocWriterEm"},{"b":"55E0DE4C6000","o":"F80512","s":"_ZN5mongo4repl12_logOpsInnerEPNS_16OperationContextERKNS_15NamespaceStringEPKPKNS_9DocWriterEmPNS_10CollectionENS0_22ReplicationCoordinator4ModeENS0_6OpTimeE"},{"b":"55E0DE4C6000","o":"F83307","s":"_ZN5mongo4repl6logOpsEPNS_16OperationContextEPKcRKNS_15NamespaceStringEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorISA_SaISA_EEEESG_b"},{"b":"55E0DE4C6000","o":"CDF238","s":"_ZN5mongo14OpObserverImpl9onInsertsEPNS_16OperationContextERKNS_15NamespaceStringEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorIS8_SaIS8_EEEESE_b"},{"b":"55E0DE4C6000","o":"9E63EA","s":"_ZN5mongo10Collection15insertDocumentsEPNS_16OperationContextEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorIS5_SaIS5_EEEESB_PNS_7OpDebugEbb"},{"b":"55E0DE4C6000","o":"9E6673","s":"_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEPNS_7OpDebugEbb"},{"b":"55E0DE4C6000","o":"BDB8B0","s":"_ZN5mongo11UpdateStage8doInsertEv"},{"b":"55E0DE4C6000","o":"BDD044","s":"_ZN5mongo11UpdateStage6doWorkEPm"},{"b":"55E0DE4C6000","o":"BA9333","s":"_ZN5mongo9PlanStage4workEPm"},{"b":"55E0DE4C6000","o":"EB362A","s":"_ZN5mongo12PlanExecutor11getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE"},{"b":"55E0DE4C6000","o":"EB3F4B","s":"_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE"},{"b":"55E0DE4C6000","o":"EB407D","s":"_ZN5mongo12PlanExecutor11executePlanEv"},{"b":"55E0DE4C6000","o":"D12DA6","s":"_ZN5mongo14performUpdatesEPNS_16OperationContextERKNS_8UpdateOpE"},{"b":"55E0DE4C6000","o":"B349FF","s":"_ZN5mongo9CmdUpdate7runImplEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERNS_14BSONObjBuilderE"},{"b":"55E0DE4C6000","o":"B3037C"},{"b":"55E0DE4C6000","o":"A6BCB5","s":"_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE"},{"b":"55E0DE4C6000","o":"A6CFA1","s":"_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE"},{"b":"55E0DE4C6000","o":"108AE00","s":"_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE"},{"b":"55E0DE4C6000","o":"C8AF12"},{"b":"55E0DE4C6000","o":"C8CF16","s":"_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE"},{"b":"55E0DE4C6000","o":"88C76D","s":"_ZN5mongo23ServiceEntryPointMongod12_sessionLoopERKSt10shared_ptrINS_9transport7SessionEE"},{"b":"55E0DE4C6000","o":"88D09D"},{"b":"55E0DE4C6000","o":"14E4251"},{"b":"7F6CF5378000","o":"7DE5"},{"b":"7F6CF4FAB000","o":"FEBAD","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.4.15", "gitVersion" : "52e5b5fbaa3a2a5b1a217f5e647b5061817475f9", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "4.14.47-56.37.amzn1.x86_64", "version" : "#1 SMP Wed Jun 6 18:49:01 UTC 2018", "machine" : "x86_64" }, "somap" : [ { "b" : "55E0DE4C6000", "elfType" : 3, "buildId" : "735C6F8FB5872DFC52902334A4BBB6D423200D89" }, { "b" : "7FFDC98DF000", "elfType" : 3, "buildId" : "2DF3D53B81C4CFBB2F14430578A041B78D5A1EE2" }, { "b" : "7F6CF6316000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "9C4EB34A346260F2A77746F4E5ED837619137DB7" }, { "b" : "7F6CF5EB8000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "EC480B38432587A9B21BFBD917EF020731EBD2CF" }, { "b" : "7F6CF5CB0000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "F2701E2A24459D5B55DF5549D585F091E7BCF07A" }, { "b" : "7F6CF5AAC000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "0E5CD5BAA5EE8BF3648A5031B088F9A78C89364F" }, { "b" : "7F6CF57AA000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "07FB92AFEF1756F093371CE60C3AE85DD3A06325" }, { "b" : "7F6CF5594000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "A03C9A80E995ED5F43077AB754A258FA0E34C3CD" }, { "b" : "7F6CF5378000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "D973C39D1900DC61D8519C653C3BC405692DE563" }, { "b" : "7F6CF4FAB000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "AF310F56618FC1EF9158973484F60942F11CC0FB" }, { "b" : "7F6CF6587000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "8402047FD4A85B3CD1142346EA06BCD6E15A82A3" }, { "b" : "7F6CF4D5E000", "path" : "/usr/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "6FBBD34B86296FDF883FE5122017EC5CD3F98ED7" }, { "b" : "7F6CF4A76000", "path" : "/usr/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "76429E6FD408BBB675798D6458F2735383710D0B" }, { "b" : "7F6CF4873000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7F6CF4640000", "path" : "/usr/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "5B2A76F1EF91EDAA0494BE680CADAFE6489326E1" }, { "b" : "7F6CF442A000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7F6CF421C000", "path" : "/usr/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "3ACB59488C6D8DE0A1F4F1B0C290A570D9E42F3D" }, { "b" : "7F6CF4019000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7F6CF3E00000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "9E5E0BF5F22DE7555BC4B9853240817147489258" }, { "b" : "7F6CF3BDF000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "F5054DC94443326819FBF3065CFDF5E4726F57EE" } ] }}
      mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x55e0dfa43371]
      mongod(+0x157C589) [0x55e0dfa42589]
      mongod(+0x157CBF6) [0x55e0dfa42bf6]
      libpthread.so.0(+0xF690) [0x7f6cf5387690]
      mongod(__wt_page_in_func+0x1010) [0x55e0e0374b40]
      mongod(__wt_row_search+0x660) [0x55e0e0395a90]
      mongod(__wt_btcur_insert+0xB72) [0x55e0e0363ee2]
      mongod(+0x1EECB14) [0x55e0e03b2b14]
      mongod(_ZN5mongo21WiredTigerRecordStore14_insertRecordsEPNS_16OperationContextEPNS_6RecordEm+0x208) [0x55e0df73c6a8]
      mongod(_ZN5mongo21WiredTigerRecordStore26insertRecordsWithDocWriterEPNS_16OperationContextEPKPKNS_9DocWriterEmPNS_8RecordIdE+0x1B5) [0x55e0df73cbb5]
      mongod(_ZN5mongo10Collection23insertDocumentsForOplogEPNS_16OperationContextEPKPKNS_9DocWriterEm+0x6F) [0x55e0deea7d9f]
      mongod(_ZN5mongo4repl12_logOpsInnerEPNS_16OperationContextERKNS_15NamespaceStringEPKPKNS_9DocWriterEmPNS_10CollectionENS0_22ReplicationCoordinator4ModeENS0_6OpTimeE+0x92) [0x55e0df446512]
      mongod(_ZN5mongo4repl6logOpsEPNS_16OperationContextEPKcRKNS_15NamespaceStringEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorISA_SaISA_EEEESG_b+0x607) [0x55e0df449307]
      mongod(_ZN5mongo14OpObserverImpl9onInsertsEPNS_16OperationContextERKNS_15NamespaceStringEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorIS8_SaIS8_EEEESE_b+0x38) [0x55e0df1a5238]
      mongod(_ZN5mongo10Collection15insertDocumentsEPNS_16OperationContextEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorIS5_SaIS5_EEEESB_PNS_7OpDebugEbb+0x45A) [0x55e0deeac3ea]
      mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEPNS_7OpDebugEbb+0x83) [0x55e0deeac673]
      mongod(_ZN5mongo11UpdateStage8doInsertEv+0x1D0) [0x55e0df0a18b0]
      mongod(_ZN5mongo11UpdateStage6doWorkEPm+0xF4) [0x55e0df0a3044]
      mongod(_ZN5mongo9PlanStage4workEPm+0x63) [0x55e0df06f333]
      mongod(_ZN5mongo12PlanExecutor11getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0x19A) [0x55e0df37962a]
      mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x4B) [0x55e0df379f4b]
      mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x6D) [0x55e0df37a07d]
      mongod(_ZN5mongo14performUpdatesEPNS_16OperationContextERKNS_8UpdateOpE+0x716) [0x55e0df1d8da6]
      mongod(_ZN5mongo9CmdUpdate7runImplEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERNS_14BSONObjBuilderE+0x4F) [0x55e0deffa9ff]
      mongod(+0xB3037C) [0x55e0deff637c]
      mongod(_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE+0x935) [0x55e0def31cb5]
      mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE+0xF81) [0x55e0def32fa1]
      mongod(_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE+0x240) [0x55e0df550e00]
      mongod(+0xC8AF12) [0x55e0df150f12]
      mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0x746) [0x55e0df152f16]
      mongod(_ZN5mongo23ServiceEntryPointMongod12_sessionLoopERKSt10shared_ptrINS_9transport7SessionEE+0x1FD) [0x55e0ded5276d]
      mongod(+0x88D09D) [0x55e0ded5309d]
      mongod(+0x14E4251) [0x55e0df9aa251]
      libpthread.so.0(+0x7DE5) [0x7f6cf537fde5]
      libc.so.6(clone+0x6D) [0x7f6cf50a9bad]
      ----- END BACKTRACE -----
       

      Attachments

        Activity

          People

            dmitry.agranat@mongodb.com Dmitry Agranat
            mike.eller@bazaarvoice.com Michael Eller
            Votes:
            0 Vote for this issue
            Watchers:
            5 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved: