Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-51899

mongo cluster crash

    XMLWordPrintableJSON

Details

    • Icon: Bug Bug
    • Resolution: Done
    • Icon: Major - P3 Major - P3
    • None
    • None
    • None

    Description

      2020-10-30T06:53:58.133+0800 F -        [thread50] terminate() called. An exception is active; attempting to gather more information
       
      2020-10-30T06:53:58.165+0800 F -        [thread50] std::exception::what(): Resource temporarily unavailable
       
      Actual exception type: std::system_error
       
       
       
      0x560b69cd1331 0x560b69cd0d15 0x560b69f707e6 0x560b69f70831 0x560b694b33af 0x560b694b3c08 0x560b68ae9dc6 0x560b6949cc66 0x560b694a1cab 0x560b694a24a2 0x560b68b1d8f4 0x560b68b1dbb0 0x560b6959af74 0x560b6959b408 0x560b68b04d32 0x560b697aa1aa 0x560b697aab6c 0x560b697aaf44 0x560b6982be09 0x560b6982c051 0x560b68b03c3d 0x560b69fe95f0 0x7fd76c65be25 0x7fd76c385bad
       
      ----- BEGIN BACKTRACE -----
       
      {"backtrace":[\{"b":"560B678EB000","o":"23E6331","s":"_ZN5mongo15printStackTraceERSo"},\{"b":"560B678EB000","o":"23E5D15"},\{"b":"560B678EB000","o":"26857E6","s":"_ZN10__cxxabiv111__terminateEPFvvE"},\{"b":"560B678EB000","o":"2685831"},\{"b":"560B678EB000","o":"1BC83AF","s":"_ZN5mongo10ThreadPool25_startWorkerThread_inlockEv"},\{"b":"560B678EB000","o":"1BC8C08","s":"_ZN5mongo10ThreadPool8scheduleESt8functionIFvvEE"},\{"b":"560B678EB000","o":"11FEDC6","s":"_ZN5mongo29ShardServerCatalogCacheLoader14getChunksSinceERKNS_15NamespaceStringENS_12ChunkVersionESt8functionIFvPNS_16OperationContextENS_10StatusWithINS_18CatalogCacheLoader26CollectionAndChangedChunksEEEEE"},\{"b":"560B678EB000","o":"1BB1C66","s":"_ZN5mongo12CatalogCache26_scheduleCollectionRefreshENS_8WithLockESt10shared_ptrINS0_17DatabaseInfoEntryEES2_INS_12ChunkManagerEERKNS_15NamespaceStringEi"},\{"b":"560B678EB000","o":"1BB6CAB","s":"_ZN5mongo12CatalogCache24getCollectionRoutingInfoEPNS_16OperationContextERKNS_15NamespaceStringE"},\{"b":"560B678EB000","o":"1BB74A2","s":"_ZN5mongo12CatalogCache42getShardedCollectionRoutingInfoWithRefreshEPNS_16OperationContextERKNS_15NamespaceStringE"},\{"b":"560B678EB000","o":"12328F4","s":"_ZN5mongo25SessionsCollectionSharded32_checkCacheForSessionsCollectionEPNS_16OperationContextE"},\{"b":"560B678EB000","o":"1232BB0","s":"_ZN5mongo25SessionsCollectionSharded23setupSessionsCollectionEPNS_16OperationContextE"},\{"b":"560B678EB000","o":"1CAFF74","s":"_ZN5mongo23LogicalSessionCacheImpl5_reapEPNS_6ClientE"},\{"b":"560B678EB000","o":"1CB0408","s":"_ZN5mongo23LogicalSessionCacheImpl13_periodicReapEPNS_6ClientE"},\{"b":"560B678EB000","o":"1219D32"},\{"b":"560B678EB000","o":"1EBF1AA","s":"_ZN4asio6detail14strand_service8dispatchINS0_7binder1ISt8functionIFvSt10error_codeEES5_EEEEvRPNS1_11strand_implERT_"},\{"b":"560B678EB000","o":"1EBFB6C","s":"_ZN4asio6detail14strand_service8dispatchINS0_17rewrapped_handlerINS0_7binder1INS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEES9_EESB_EEEEvRPNS1_11strand_implERT_"},\{"b":"560B678EB000","o":"1EBFF44","s":"_ZN4asio6detail12wait_handlerINS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEEE11do_completeEPvPNS0_19scheduler_operationERKS6_m"},\{"b":"560B678EB000","o":"1F40E09","s":"_ZN4asio6detail9scheduler10do_run_oneERNS0_27conditionally_enabled_mutex11scoped_lockERNS0_21scheduler_thread_infoERKSt10error_code"},\{"b":"560B678EB000","o":"1F41051","s":"_ZN4asio6detail9scheduler3runERSt10error_code"},\{"b":"560B678EB000","o":"1218C3D"},\{"b":"560B678EB000","o":"26FE5F0"},\{"b":"7FD76C654000","o":"7E25"},\{"b":"7FD76C287000","o":"FEBAD","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.6.7-1.5", "gitVersion" : "f4cbafeaa1be7aa014838178f61de5a44a850cfe", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "[3.10.0-862.el7.x86_64|https://jira.mongodb.org/secure/3.10.0-862.el7.x86_64]", "version" : "#1 SMP Fri Apr 20 16:44:24 UTC 2018", "machine" : "x86_64" }, "somap" : [ \{ "b" : "560B678EB000", "elfType" : 3, "buildId" : "3C59BF36EF2D7BEAC27661CAE934C8D1AF7B77FF" }, \{ "b" : "7FFFBE8B9000", "elfType" : 3, "buildId" : "463512F624A152723F0EFEED311DB2A05ACEF3D6" }, { "b" : "7FD76DEAD000", "path" : "/lib64/[libz.so.1|https://jira.mongodb.org/secure/libz.so.1]", "elfType" : 3, "buildId" : "EA8E45DC8E395CC5E26890470112D97A1F1E0B65" }, { "b" : "7FD76DC9D000", "path" : "/lib64/[libbz2.so.1|https://jira.mongodb.org/secure/libbz2.so.1]", "elfType" : 3, "buildId" : "0C85C0386F0CF41EA39969CF7F58A558D1AD3235" }, { "b" : "7FD76DA80000", "path" : "/lib64/[libsasl2.so.3|https://jira.mongodb.org/secure/libsasl2.so.3]", "elfType" : 3, "buildId" : "E2F2017F821DD1B9D307DA1A9B8014F2941AEB7B" }, { "b" : "7FD76D867000", "path" : "/lib64/[libresolv.so.2|https://jira.mongodb.org/secure/libresolv.so.2]", "elfType" : 3, "buildId" : "2BDC2B6FF0B2C204CCE34D139A9EADA0272EB070" }, { "b" : "7FD76D406000", "path" : "/lib64/[libcrypto.so.10|https://jira.mongodb.org/secure/libcrypto.so.10]", "elfType" : 3, "buildId" : "67CEB4EDD36BFE0EB31CD92DA2694ACA5377A599" }, { "b" : "7FD76D194000", "path" : "/lib64/[libssl.so.10|https://jira.mongodb.org/secure/libssl.so.10]", "elfType" : 3, "buildId" : "6468A4E28A19CDD885A3CBC30E009589CA4C2E92" }, { "b" : "7FD76CF90000", "path" : "/lib64/[libdl.so.2|https://jira.mongodb.org/secure/libdl.so.2]", "elfType" : 3, "buildId" : "5CDB5A56336E7E2BD14FFA189411E44A834AFCD8" }, { "b" : "7FD76CD88000", "path" : "/lib64/[librt.so.1|https://jira.mongodb.org/secure/librt.so.1]", "elfType" : 3, "buildId" : "D33989EC31EFE745EB0D3B68A92D19E77D7DDFDA" }, { "b" : "7FD76CA86000", "path" : "/lib64/[libm.so.6|https://jira.mongodb.org/secure/libm.so.6]", "elfType" : 3, "buildId" : "F4CAE74047F9AA2D5A71FDEC67C4285D75753EBA" }, { "b" : "7FD76C870000", "path" : "/lib64/[libgcc_s.so.1|https://jira.mongodb.org/secure/libgcc_s.so.1]", "elfType" : 3, "buildId" : "7D59364518A4455DBF57E2BE46E9436161ECC487" }, { "b" : "7FD76C654000", "path" : "/lib64/[libpthread.so.0|https://jira.mongodb.org/secure/libpthread.so.0]", "elfType" : 3, "buildId" : "F4C04BCE85D2D269D0A2AF4972FC69805B50345B" }, { "b" : "7FD76C287000", "path" : "/lib64/[libc.so.6|https://jira.mongodb.org/secure/libc.so.6]", "elfType" : 3, "buildId" : "CB4B7554D1ADBEF2F001142DD6F0A5139FC9AA69" }, { "b" : "7FD76E0C3000", "path" : "/lib64/[ld-linux-x86-64.so.2|https://jira.mongodb.org/secure/ld-linux-x86-64.so.2]", "elfType" : 3, "buildId" : "D266B1F6650927E18108323BCCA8F7B68E68EB92" }, { "b" : "7FD76C050000", "path" : "/lib64/[libcrypt.so.1|https://jira.mongodb.org/secure/libcrypt.so.1]", "elfType" : 3, "buildId" : "774AE2495620D6BA59768814D07C450844864A1F" }, { "b" : "7FD76BE03000", "path" : "/lib64/[libgssapi_krb5.so.2|https://jira.mongodb.org/secure/libgssapi_krb5.so.2]", "elfType" : 3, "buildId" : "39E6427542C05B1019EC312345239164CCFE84DF" }, { "b" : "7FD76BB1B000", "path" : "/lib64/[libkrb5.so.3|https://jira.mongodb.org/secure/libkrb5.so.3]", "elfType" : 3, "buildId" : "36971F0AF2B69B25C7F9589CDE3A82D34A60C591" }, { "b" : "7FD76B8E8000", "path" : "/lib64/[libk5crypto.so.3|https://jira.mongodb.org/secure/libk5crypto.so.3]", "elfType" : 3, "buildId" : "9891D2B285C6D5E761DFE7AEE8BAD629EA73D061" }, { "b" : "7FD76B6E4000", "path" : "/lib64/[libcom_err.so.2|https://jira.mongodb.org/secure/libcom_err.so.2]", "elfType" : 3, "buildId" : "5025A7A2E8AECA9ACB996121370821B9CAA9E397" }, { "b" : "7FD76B4D6000", "path" : "/lib64/[libkrb5support.so.0|https://jira.mongodb.org/secure/libkrb5support.so.0]", "elfType" : 3, "buildId" : "AA19B311D788A700456C8A7E1C727F8609CFB302" }, { "b" : "7FD76B2D3000", "path" : "/lib64/[libfreebl3.so|https://jira.mongodb.org/secure/libfreebl3.so]", "elfType" : 3, "buildId" : "0F799763563D51F3DF612B227C707A8FDAEA59D3" }, { "b" : "7FD76B0CF000", "path" : "/lib64/[libkeyutils.so.1|https://jira.mongodb.org/secure/libkeyutils.so.1]", "elfType" : 3, "buildId" : "2E01D5AC08C1280D013AAB96B292AC58BC30A263" }, { "b" : "7FD76AEA8000", "path" : "/lib64/[libselinux.so.1|https://jira.mongodb.org/secure/libselinux.so.1]", "elfType" : 3, "buildId" : "1B0C0DC4F9054C5922082921C1447739F27AC8F4" }, { "b" : "7FD76AC46000", "path" : "/lib64/[libpcre.so.1|https://jira.mongodb.org/secure/libpcre.so.1]", "elfType" : 3, "buildId" : "9CA3D11F018BEEB719CDB34BE800BF1641350D0A" } ] }}
       
      mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x560b69cd1331]
       
      mongod(+0x23E5D15) [0x560b69cd0d15]
       
      mongod(_ZN10__cxxabiv111__terminateEPFvvE+0x6) [0x560b69f707e6]
       
      mongod(+0x2685831) [0x560b69f70831]
       
      mongod(_ZN5mongo10ThreadPool25_startWorkerThread_inlockEv+0x99F) [0x560b694b33af]
       
      mongod(_ZN5mongo10ThreadPool8scheduleESt8functionIFvvEE+0x398) [0x560b694b3c08]
       
      mongod(_ZN5mongo29ShardServerCatalogCacheLoader14getChunksSinceERKNS_15NamespaceStringENS_12ChunkVersionESt8functionIFvPNS_16OperationContextENS_10StatusWithINS_18CatalogCacheLoader26CollectionAndChangedChunksEEEEE+0x2A6) [0x560b68ae9dc6]
       
      mongod(_ZN5mongo12CatalogCache26_scheduleCollectionRefreshENS_8WithLockESt10shared_ptrINS0_17DatabaseInfoEntryEES2_INS_12ChunkManagerEERKNS_15NamespaceStringEi+0xCE6) [0x560b6949cc66]
       
      mongod(_ZN5mongo12CatalogCache24getCollectionRoutingInfoEPNS_16OperationContextERKNS_15NamespaceStringE+0x93B) [0x560b694a1cab]
       
      mongod(_ZN5mongo12CatalogCache42getShardedCollectionRoutingInfoWithRefreshEPNS_16OperationContextERKNS_15NamespaceStringE+0x52) [0x560b694a24a2]
       
      mongod(_ZN5mongo25SessionsCollectionSharded32_checkCacheForSessionsCollectionEPNS_16OperationContextE+0x64) [0x560b68b1d8f4]
       
      mongod(_ZN5mongo25SessionsCollectionSharded23setupSessionsCollectionEPNS_16OperationContextE+0x20) [0x560b68b1dbb0]
       
      mongod(_ZN5mongo23LogicalSessionCacheImpl5_reapEPNS_6ClientE+0xC4) [0x560b6959af74]
       
      mongod(_ZN5mongo23LogicalSessionCacheImpl13_periodicReapEPNS_6ClientE+0x38) [0x560b6959b408]
       
      mongod(+0x1219D32) [0x560b68b04d32]
       
      mongod(_ZN4asio6detail14strand_service8dispatchINS0_7binder1ISt8functionIFvSt10error_codeEES5_EEEEvRPNS1_11strand_implERT_+0x7A) [0x560b697aa1aa]
       
      mongod(_ZN4asio6detail14strand_service8dispatchINS0_17rewrapped_handlerINS0_7binder1INS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEES9_EESB_EEEEvRPNS1_11strand_implERT_+0x3AC) [0x560b697aab6c]
       
      mongod(_ZN4asio6detail12wait_handlerINS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEEE11do_completeEPvPNS0_19scheduler_operationERKS6_m+0x164) [0x560b697aaf44]
       
      mongod(_ZN4asio6detail9scheduler10do_run_oneERNS0_27conditionally_enabled_mutex11scoped_lockERNS0_21scheduler_thread_infoERKSt10error_code+0x389) [0x560b6982be09]
       
      mongod(_ZN4asio6detail9scheduler3runERSt10error_code+0xD1) [0x560b6982c051]
       
      mongod(+0x1218C3D) [0x560b68b03c3d]
       
      mongod(+0x26FE5F0) [0x560b69fe95f0]
       
       [libpthread.so.0(+0x7E25|https://jira.mongodb.org/secure/libpthread.so.0(+0x7E25]) [0x7fd76c65be25]
       
       [libc.so.6(clone+0x6D|https://jira.mongodb.org/secure/libc.so.6(clone+0x6D]) [0x7fd76c385bad]
       
      -----  END BACKTRACE  -----
      

      Attachments

        Activity

          People

            eric.sedor@mongodb.com Eric Sedor
            1994lwx@gmail.com 文熙 刘
            Votes:
            0 Vote for this issue
            Watchers:
            7 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved: