[SERVER-51899] mongo cluster crash Created: 30/Oct/20  Updated: 15/Mar/21  Resolved: 15/Mar/21

Status: Closed
Project: Core Server
Component/s: None
Affects Version/s: None
Fix Version/s: None

Type: Bug Priority: Major - P3
Reporter: 文熙 刘 Assignee: Eric Sedor
Resolution: Done Votes: 0
Labels: Bug
Remaining Estimate: Not Specified
Time Spent: Not Specified
Original Estimate: Not Specified

Participants:

 Description   

2020-10-30T06:53:58.133+0800 F -        [thread50] terminate() called. An exception is active; attempting to gather more information
 
2020-10-30T06:53:58.165+0800 F -        [thread50] std::exception::what(): Resource temporarily unavailable
 
Actual exception type: std::system_error
 
 
 
0x560b69cd1331 0x560b69cd0d15 0x560b69f707e6 0x560b69f70831 0x560b694b33af 0x560b694b3c08 0x560b68ae9dc6 0x560b6949cc66 0x560b694a1cab 0x560b694a24a2 0x560b68b1d8f4 0x560b68b1dbb0 0x560b6959af74 0x560b6959b408 0x560b68b04d32 0x560b697aa1aa 0x560b697aab6c 0x560b697aaf44 0x560b6982be09 0x560b6982c051 0x560b68b03c3d 0x560b69fe95f0 0x7fd76c65be25 0x7fd76c385bad
 
----- BEGIN BACKTRACE -----
 
{"backtrace":[\{"b":"560B678EB000","o":"23E6331","s":"_ZN5mongo15printStackTraceERSo"},\{"b":"560B678EB000","o":"23E5D15"},\{"b":"560B678EB000","o":"26857E6","s":"_ZN10__cxxabiv111__terminateEPFvvE"},\{"b":"560B678EB000","o":"2685831"},\{"b":"560B678EB000","o":"1BC83AF","s":"_ZN5mongo10ThreadPool25_startWorkerThread_inlockEv"},\{"b":"560B678EB000","o":"1BC8C08","s":"_ZN5mongo10ThreadPool8scheduleESt8functionIFvvEE"},\{"b":"560B678EB000","o":"11FEDC6","s":"_ZN5mongo29ShardServerCatalogCacheLoader14getChunksSinceERKNS_15NamespaceStringENS_12ChunkVersionESt8functionIFvPNS_16OperationContextENS_10StatusWithINS_18CatalogCacheLoader26CollectionAndChangedChunksEEEEE"},\{"b":"560B678EB000","o":"1BB1C66","s":"_ZN5mongo12CatalogCache26_scheduleCollectionRefreshENS_8WithLockESt10shared_ptrINS0_17DatabaseInfoEntryEES2_INS_12ChunkManagerEERKNS_15NamespaceStringEi"},\{"b":"560B678EB000","o":"1BB6CAB","s":"_ZN5mongo12CatalogCache24getCollectionRoutingInfoEPNS_16OperationContextERKNS_15NamespaceStringE"},\{"b":"560B678EB000","o":"1BB74A2","s":"_ZN5mongo12CatalogCache42getShardedCollectionRoutingInfoWithRefreshEPNS_16OperationContextERKNS_15NamespaceStringE"},\{"b":"560B678EB000","o":"12328F4","s":"_ZN5mongo25SessionsCollectionSharded32_checkCacheForSessionsCollectionEPNS_16OperationContextE"},\{"b":"560B678EB000","o":"1232BB0","s":"_ZN5mongo25SessionsCollectionSharded23setupSessionsCollectionEPNS_16OperationContextE"},\{"b":"560B678EB000","o":"1CAFF74","s":"_ZN5mongo23LogicalSessionCacheImpl5_reapEPNS_6ClientE"},\{"b":"560B678EB000","o":"1CB0408","s":"_ZN5mongo23LogicalSessionCacheImpl13_periodicReapEPNS_6ClientE"},\{"b":"560B678EB000","o":"1219D32"},\{"b":"560B678EB000","o":"1EBF1AA","s":"_ZN4asio6detail14strand_service8dispatchINS0_7binder1ISt8functionIFvSt10error_codeEES5_EEEEvRPNS1_11strand_implERT_"},\{"b":"560B678EB000","o":"1EBFB6C","s":"_ZN4asio6detail14strand_service8dispatchINS0_17rewrapped_handlerINS0_7binder1INS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEES9_EESB_EEEEvRPNS1_11strand_implERT_"},\{"b":"560B678EB000","o":"1EBFF44","s":"_ZN4asio6detail12wait_handlerINS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEEE11do_completeEPvPNS0_19scheduler_operationERKS6_m"},\{"b":"560B678EB000","o":"1F40E09","s":"_ZN4asio6detail9scheduler10do_run_oneERNS0_27conditionally_enabled_mutex11scoped_lockERNS0_21scheduler_thread_infoERKSt10error_code"},\{"b":"560B678EB000","o":"1F41051","s":"_ZN4asio6detail9scheduler3runERSt10error_code"},\{"b":"560B678EB000","o":"1218C3D"},\{"b":"560B678EB000","o":"26FE5F0"},\{"b":"7FD76C654000","o":"7E25"},\{"b":"7FD76C287000","o":"FEBAD","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.6.7-1.5", "gitVersion" : "f4cbafeaa1be7aa014838178f61de5a44a850cfe", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "[3.10.0-862.el7.x86_64|https://jira.mongodb.org/secure/3.10.0-862.el7.x86_64]", "version" : "#1 SMP Fri Apr 20 16:44:24 UTC 2018", "machine" : "x86_64" }, "somap" : [ \{ "b" : "560B678EB000", "elfType" : 3, "buildId" : "3C59BF36EF2D7BEAC27661CAE934C8D1AF7B77FF" }, \{ "b" : "7FFFBE8B9000", "elfType" : 3, "buildId" : "463512F624A152723F0EFEED311DB2A05ACEF3D6" }, { "b" : "7FD76DEAD000", "path" : "/lib64/[libz.so.1|https://jira.mongodb.org/secure/libz.so.1]", "elfType" : 3, "buildId" : "EA8E45DC8E395CC5E26890470112D97A1F1E0B65" }, { "b" : "7FD76DC9D000", "path" : "/lib64/[libbz2.so.1|https://jira.mongodb.org/secure/libbz2.so.1]", "elfType" : 3, "buildId" : "0C85C0386F0CF41EA39969CF7F58A558D1AD3235" }, { "b" : "7FD76DA80000", "path" : "/lib64/[libsasl2.so.3|https://jira.mongodb.org/secure/libsasl2.so.3]", "elfType" : 3, "buildId" : "E2F2017F821DD1B9D307DA1A9B8014F2941AEB7B" }, { "b" : "7FD76D867000", "path" : "/lib64/[libresolv.so.2|https://jira.mongodb.org/secure/libresolv.so.2]", "elfType" : 3, "buildId" : "2BDC2B6FF0B2C204CCE34D139A9EADA0272EB070" }, { "b" : "7FD76D406000", "path" : "/lib64/[libcrypto.so.10|https://jira.mongodb.org/secure/libcrypto.so.10]", "elfType" : 3, "buildId" : "67CEB4EDD36BFE0EB31CD92DA2694ACA5377A599" }, { "b" : "7FD76D194000", "path" : "/lib64/[libssl.so.10|https://jira.mongodb.org/secure/libssl.so.10]", "elfType" : 3, "buildId" : "6468A4E28A19CDD885A3CBC30E009589CA4C2E92" }, { "b" : "7FD76CF90000", "path" : "/lib64/[libdl.so.2|https://jira.mongodb.org/secure/libdl.so.2]", "elfType" : 3, "buildId" : "5CDB5A56336E7E2BD14FFA189411E44A834AFCD8" }, { "b" : "7FD76CD88000", "path" : "/lib64/[librt.so.1|https://jira.mongodb.org/secure/librt.so.1]", "elfType" : 3, "buildId" : "D33989EC31EFE745EB0D3B68A92D19E77D7DDFDA" }, { "b" : "7FD76CA86000", "path" : "/lib64/[libm.so.6|https://jira.mongodb.org/secure/libm.so.6]", "elfType" : 3, "buildId" : "F4CAE74047F9AA2D5A71FDEC67C4285D75753EBA" }, { "b" : "7FD76C870000", "path" : "/lib64/[libgcc_s.so.1|https://jira.mongodb.org/secure/libgcc_s.so.1]", "elfType" : 3, "buildId" : "7D59364518A4455DBF57E2BE46E9436161ECC487" }, { "b" : "7FD76C654000", "path" : "/lib64/[libpthread.so.0|https://jira.mongodb.org/secure/libpthread.so.0]", "elfType" : 3, "buildId" : "F4C04BCE85D2D269D0A2AF4972FC69805B50345B" }, { "b" : "7FD76C287000", "path" : "/lib64/[libc.so.6|https://jira.mongodb.org/secure/libc.so.6]", "elfType" : 3, "buildId" : "CB4B7554D1ADBEF2F001142DD6F0A5139FC9AA69" }, { "b" : "7FD76E0C3000", "path" : "/lib64/[ld-linux-x86-64.so.2|https://jira.mongodb.org/secure/ld-linux-x86-64.so.2]", "elfType" : 3, "buildId" : "D266B1F6650927E18108323BCCA8F7B68E68EB92" }, { "b" : "7FD76C050000", "path" : "/lib64/[libcrypt.so.1|https://jira.mongodb.org/secure/libcrypt.so.1]", "elfType" : 3, "buildId" : "774AE2495620D6BA59768814D07C450844864A1F" }, { "b" : "7FD76BE03000", "path" : "/lib64/[libgssapi_krb5.so.2|https://jira.mongodb.org/secure/libgssapi_krb5.so.2]", "elfType" : 3, "buildId" : "39E6427542C05B1019EC312345239164CCFE84DF" }, { "b" : "7FD76BB1B000", "path" : "/lib64/[libkrb5.so.3|https://jira.mongodb.org/secure/libkrb5.so.3]", "elfType" : 3, "buildId" : "36971F0AF2B69B25C7F9589CDE3A82D34A60C591" }, { "b" : "7FD76B8E8000", "path" : "/lib64/[libk5crypto.so.3|https://jira.mongodb.org/secure/libk5crypto.so.3]", "elfType" : 3, "buildId" : "9891D2B285C6D5E761DFE7AEE8BAD629EA73D061" }, { "b" : "7FD76B6E4000", "path" : "/lib64/[libcom_err.so.2|https://jira.mongodb.org/secure/libcom_err.so.2]", "elfType" : 3, "buildId" : "5025A7A2E8AECA9ACB996121370821B9CAA9E397" }, { "b" : "7FD76B4D6000", "path" : "/lib64/[libkrb5support.so.0|https://jira.mongodb.org/secure/libkrb5support.so.0]", "elfType" : 3, "buildId" : "AA19B311D788A700456C8A7E1C727F8609CFB302" }, { "b" : "7FD76B2D3000", "path" : "/lib64/[libfreebl3.so|https://jira.mongodb.org/secure/libfreebl3.so]", "elfType" : 3, "buildId" : "0F799763563D51F3DF612B227C707A8FDAEA59D3" }, { "b" : "7FD76B0CF000", "path" : "/lib64/[libkeyutils.so.1|https://jira.mongodb.org/secure/libkeyutils.so.1]", "elfType" : 3, "buildId" : "2E01D5AC08C1280D013AAB96B292AC58BC30A263" }, { "b" : "7FD76AEA8000", "path" : "/lib64/[libselinux.so.1|https://jira.mongodb.org/secure/libselinux.so.1]", "elfType" : 3, "buildId" : "1B0C0DC4F9054C5922082921C1447739F27AC8F4" }, { "b" : "7FD76AC46000", "path" : "/lib64/[libpcre.so.1|https://jira.mongodb.org/secure/libpcre.so.1]", "elfType" : 3, "buildId" : "9CA3D11F018BEEB719CDB34BE800BF1641350D0A" } ] }}
 
mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x560b69cd1331]
 
mongod(+0x23E5D15) [0x560b69cd0d15]
 
mongod(_ZN10__cxxabiv111__terminateEPFvvE+0x6) [0x560b69f707e6]
 
mongod(+0x2685831) [0x560b69f70831]
 
mongod(_ZN5mongo10ThreadPool25_startWorkerThread_inlockEv+0x99F) [0x560b694b33af]
 
mongod(_ZN5mongo10ThreadPool8scheduleESt8functionIFvvEE+0x398) [0x560b694b3c08]
 
mongod(_ZN5mongo29ShardServerCatalogCacheLoader14getChunksSinceERKNS_15NamespaceStringENS_12ChunkVersionESt8functionIFvPNS_16OperationContextENS_10StatusWithINS_18CatalogCacheLoader26CollectionAndChangedChunksEEEEE+0x2A6) [0x560b68ae9dc6]
 
mongod(_ZN5mongo12CatalogCache26_scheduleCollectionRefreshENS_8WithLockESt10shared_ptrINS0_17DatabaseInfoEntryEES2_INS_12ChunkManagerEERKNS_15NamespaceStringEi+0xCE6) [0x560b6949cc66]
 
mongod(_ZN5mongo12CatalogCache24getCollectionRoutingInfoEPNS_16OperationContextERKNS_15NamespaceStringE+0x93B) [0x560b694a1cab]
 
mongod(_ZN5mongo12CatalogCache42getShardedCollectionRoutingInfoWithRefreshEPNS_16OperationContextERKNS_15NamespaceStringE+0x52) [0x560b694a24a2]
 
mongod(_ZN5mongo25SessionsCollectionSharded32_checkCacheForSessionsCollectionEPNS_16OperationContextE+0x64) [0x560b68b1d8f4]
 
mongod(_ZN5mongo25SessionsCollectionSharded23setupSessionsCollectionEPNS_16OperationContextE+0x20) [0x560b68b1dbb0]
 
mongod(_ZN5mongo23LogicalSessionCacheImpl5_reapEPNS_6ClientE+0xC4) [0x560b6959af74]
 
mongod(_ZN5mongo23LogicalSessionCacheImpl13_periodicReapEPNS_6ClientE+0x38) [0x560b6959b408]
 
mongod(+0x1219D32) [0x560b68b04d32]
 
mongod(_ZN4asio6detail14strand_service8dispatchINS0_7binder1ISt8functionIFvSt10error_codeEES5_EEEEvRPNS1_11strand_implERT_+0x7A) [0x560b697aa1aa]
 
mongod(_ZN4asio6detail14strand_service8dispatchINS0_17rewrapped_handlerINS0_7binder1INS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEES9_EESB_EEEEvRPNS1_11strand_implERT_+0x3AC) [0x560b697aab6c]
 
mongod(_ZN4asio6detail12wait_handlerINS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEEE11do_completeEPvPNS0_19scheduler_operationERKS6_m+0x164) [0x560b697aaf44]
 
mongod(_ZN4asio6detail9scheduler10do_run_oneERNS0_27conditionally_enabled_mutex11scoped_lockERNS0_21scheduler_thread_infoERKSt10error_code+0x389) [0x560b6982be09]
 
mongod(_ZN4asio6detail9scheduler3runERSt10error_code+0xD1) [0x560b6982c051]
 
mongod(+0x1218C3D) [0x560b68b03c3d]
 
mongod(+0x26FE5F0) [0x560b69fe95f0]
 
 [libpthread.so.0(+0x7E25|https://jira.mongodb.org/secure/libpthread.so.0(+0x7E25]) [0x7fd76c65be25]
 
 [libc.so.6(clone+0x6D|https://jira.mongodb.org/secure/libc.so.6(clone+0x6D]) [0x7fd76c385bad]
 
-----  END BACKTRACE  -----



 Comments   
Comment by Eric Sedor [ 15/Mar/21 ]

Hi,

We haven’t heard back from you for some time, so I’m going to close this ticket. If this is still an issue for you, please provide additional information and we will reopen the ticket.

Regards,
Eric

Comment by Eric Sedor [ 19/Nov/20 ]

Hi,

We still need additional information to diagnose the problem. If this is still an issue for you, would you please clarify the above?

Thanks,
Eric

Comment by Eric Sedor [ 30/Oct/20 ]

Hello,

Can you please provide the mongodb version and a description/timeline of the events leading up to issuing the command to shut down the server? Logs leading up to the crash will be helpful as well.

Additionally, the ticket summary mentions a cluster crash. Did just one server crash or were there more?

Thanks,
Eric

Generated at Thu Feb 08 05:26:46 UTC 2024 using Jira 9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66.