Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-34206

All replica nodes crash

    • Type: Icon: Bug Bug
    • Resolution: Duplicate
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: 3.6.3
    • Component/s: Replication, Sharding
    • Labels:
      None
    • ALL
    • Hide

      I upgraded from 3.4.x to 3.6.3 and now all 3 all my replica nodes crash about 5 minutes after starting

      Show
      I upgraded from 3.4.x to 3.6.3 and now all 3 all my replica nodes crash about 5 minutes after starting

      {{2018-03-30T13:38:24.558+0000 F - [thread116] Invalid access at address: 0x18
      2018-03-30T13:38:24.574+0000 F - [thread116] Got signal: 11 (Segmentation fault).

      0x55f714b69071 0x55f714b68289 0x55f714b688f6 0x7f70d11a6390 0x7f70d119ed44 0x55f714343456 0x55f7143481e1 0x55f7139d727e 0x55f7139d75a0 0x55f714446cda 0x55f714448188 0x55f7139be622 0x55f71464334a 0x55f714643d0c 0x55f7146440e4 0x55f7146b9b59 0x55f7146b9da1 0x55f7139bd52d 0x55f714c78170
      0x7f70d119c6ba 0x7f70d0ed241d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[

      {"b":"55F712957000","o":"2212071","s":"_ZN5mongo15printStackTraceERSo"}

      ,

      {"b":"55F712957000","o":"2211289"}

      ,

      {"b":"55F712957000","o":"22118F6"}

      ,

      {"b":"7F70D1195000","o":"11390"}

      ,

      {"b":"7F70D1195000","o":"9D44","s":"pthread_mutex_lock"}

      ,

      {"b":"55F712957000","o":"19EC456","s":"_ ZN5mongo12CatalogCache27invalidateShardedCollectionERKNS_15NamespaceStringE"}

      ,

      {"b":"55F712957000","o":"19F11E1","s":"_ZN5mongo12CatalogCache42getShardedCollectionRoutingInfoWithRefreshEPNS_16OperationContextERKNS_15NamespaceStringE"}

      ,

      {"b":"55F712957000","o":"108027E","s":"_ZN5mongo25Se ssionsCollectionSharded32_checkCacheForSessionsCollectionEPNS_16OperationContextE"}

      ,

      {"b":"55F712957000","o":"10805A0","s":"_ZN5mongo25SessionsCollectionSharded23setupSessionsCollectionEPNS_16OperationContextE"}

      ,

      {"b":"55F712957000","o":"1AEFCDA","s":"_ZN5mongo23LogicalSessionCacheImpl8_ refreshEPNS_6ClientE"}

      ,

      {"b":"55F712957000","o":"1AF1188","s":"_ZN5mongo23LogicalSessionCacheImpl16_periodicRefreshEPNS_6ClientE"}

      ,

      {"b":"55F712957000","o":"1067622"}

      ,

      {"b":"55F712957000","o":"1CEC34A","s":"_ZN4asio6detail14strand_service8dispatchINS0_7binder1ISt8functionIFvSt10error_code EES5_EEEEvRPNS1_11strand_implERT_"}

      ,

      {"b":"55F712957000","o":"1CECD0C","s":"_ZN4asio6detail14strand_service8dispatchINS0_17rewrapped_handlerINS0_7binder1INS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEES9_EESB_EEEEvRPNS1_11st rand_implERT_"}

      ,

      {"b":"55F712957000","o":"1CED0E4","s":"_ZN4asio6detail12wait_handlerINS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEEE11do_completeEPvPNS0_19scheduler_operationERKS6_m"}

      ,

      {"b":"55F712957000","o":"1D62B59","s": "_ZN4asio6detail9scheduler10do_run_oneERNS0_27conditionally_enabled_mutex11scoped_lockERNS0_21scheduler_thread_infoERKSt10error_code"}

      ,

      {"b":"55F712957000","o":"1D62DA1","s":"_ZN4asio6detail9scheduler3runERSt10error_code"}

      ,

      {"b":"55F712957000","o":"106652D"}

      ,

      {"b":"55F712957000","o":"2321 170"}

      ,

      {"b":"7F70D1195000","o":"76BA"}

      ,

      {"b":"7F70D0DCB000","o":"10741D","s":"clone"}

      ],"processInfo":{ "mongodbVersion" : "3.6.3", "gitVersion" : "9586e557d54ef70f9ca4b43c26892cd55257e1a5", "compiledModules" : [], "uname" :

      { "sysname" : "Linux", "release" : "4.14.12-x86_64-linode92", "v ersion" : "#1 SMP Fri Jan 5 15:34:44 UTC 2018", "machine" : "x86_64" }

      , "somap" : [

      { "b" : "55F712957000", "elfType" : 3, "buildId" : "7A0DE17E89E4F8A85A88DD691D5A6B4053CB0C42" }

      ,

      { "b" : "7FFDEA9E1000", "elfType" : 3, "buildId" : "5B9F82F9B3254BD74F9E4464058886A5700FE623" }

      ,

      { "b" : "7F70D238A000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "6EF73266978476EF9F2FD2CF31E57F4597CB74F8" }

      ,

      { "b" : "7F70D2121000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "5A9B9023B6E9ABCEE21DADDB95EF096F505E9E67" }

      ,

      { "b" : "7F70D1CDD000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "B467957BF7E767DD5C0CA92213C2DAF0182765D2" }

      ,

      { "b" : "7F70D1AD5000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "89C34D7A182387D76D5CDA1F7718F5D58824DFB3 " }

      ,

      { "b" : "7F70D18D1000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "8CC8D0D119B142D839800BFF71FB71E73AEA7BD4" }

      ,

      { "b" : "7F70D15C8000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "DFB85DE42DAFFD09640C8FE377D572DE3E168920" }

      ,

      { "b" : "7F70D13B2000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "68220AE2C65D65C1B6AAA12FA6765A6EC2F5F434" }

      ,

      { "b" : "7F70D1195000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "CE17E023542265FC11D9BC8F534BB4F070493 D30" }

      ,

      { "b" : "7F70D0DCB000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "B5381A457906D279073822A5CEB24C4BFEF94DDB" }

      ,

      { "b" : "7F70D25A5000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "5D7B6259552275A3C17BD4C3FD05F5A6BF40CAA5" }

      ]
      }}
      mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x55f714b69071]
      mongod(+0x2211289) [0x55f714b68289]
      mongod(+0x22118F6) [0x55f714b688f6]
      libpthread.so.0(+0x11390) [0x7f70d11a6390]
      libpthread.so.0(pthread_mutex_lock+0x4) [0x7f70d119ed44]
      mongod(_ZN5mongo12CatalogCache27invalidateShardedCollectionERKNS_15NamespaceStringE+0x46) [0x55f714343456]
      mongod(_ZN5mongo12CatalogCache42getShardedCollectionRoutingInfoWithRefreshEPNS_16OperationContextERKNS_15NamespaceStringE+0x41) [0x55f7143481e1]
      mongod(_ZN5mongo25SessionsCollectionSharded32_checkCacheForSessionsCollectionEPNS_16OperationContextE+0x10E) [0x55f7139d727e]
      mongod(_ZN5mongo25SessionsCollectionSharded23setupSessionsCollectionEPNS_16OperationContextE+0x20) [0x55f7139d75a0]
      mongod(_ZN5mongo23LogicalSessionCacheImpl8_refreshEPNS_6ClientE+0x12A) [0x55f714446cda]
      mongod(_ZN5mongo23LogicalSessionCacheImpl16_periodicRefreshEPNS_6ClientE+0x28) [0x55f714448188]
      mongod(+0x1067622) [0x55f7139be622]
      mongod(ZN4asio6detail14strand_service8dispatchINS0_7binder1ISt8functionIFvSt10error_codeEES5_EEEEvRPNS1_11strand_implERT+0x7A) [0x55f71464334a]
      mongod(ZN4asio6detail14strand_service8dispatchINS0_17rewrapped_handlerINS0_7binder1INS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEES9_EESB_EEEEvRPNS1_11strand_implERT+0x3AC) [0x55f714643d0c]
      mongod(_ZN4asio6detail12wait_handlerINS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEEE11do_completeEPvPNS0_19scheduler_operationERKS6_m+0x164) [0x55f7146440e4]
      mongod(_ZN4asio6detail9scheduler10do_run_oneERNS0_27conditionally_enabled_mutex11scoped_lockERNS0_21scheduler_thread_infoERKSt10error_code+0x389) [0x55f7146b9b59]
      mongod(_ZN4asio6detail9scheduler3runERSt10error_code+0xD1) [0x55f7146b9da1]
      mongod(+0x106652D) [0x55f7139bd52d]
      ...skipping...}}

            Assignee:
            kelsey.schubert@mongodb.com Kelsey Schubert
            Reporter:
            travis@bryx.com Travis Brown
            Votes:
            0 Vote for this issue
            Watchers:
            8 Start watching this issue

              Created:
              Updated:
              Resolved: