Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-41235

Invariant failure !finished() src/mongo/db/concurrency/fast_map_noalloc.h 94

    • Type: Icon: Bug Bug
    • Resolution: Incomplete
    • Priority: Icon: Minor - P4 Minor - P4
    • None
    • Affects Version/s: 3.6.4
    • Component/s: None
    • Labels:
      None
    • Environment:
      Running on AWS T2.medium instance
      (2 vCPU, 4GB Mem)

      OS - Amazon Linux
    • ALL

      We've been running these DBs for well over a couple of years now, updated to mongo 3.6.4 roughly 6 month ago. One of the members of the replica set appears to have been taken offline from the below error, we're unable to reproduce as we can't identify what request(s) occured at this time.

      [conn7261063] received client metadata from 192.168.0.30:39234 conn7261063: { driver: { name: "PyMongo", version: "3.5.1" }, os: { type: "Linux", name: "", architecture: "x86_64", version: "4.4.41-36.55.amzn1.x86_64" }, platform: "CPython 2.7.15.final.0" }
      2019-05-17T05:39:56.225+0100 I NETWORK [conn7261063] end connection 192.168.0.30:39234 (27 connections now open)
      2019-05-17T05:39:58.760+0100 F - [thread42] Invariant failure !finished() src/mongo/db/concurrency/fast_map_noalloc.h 94
      2019-05-17T05:39:58.760+0100 F - [thread42]
      ***aborting after invariant() failure
      
      2019-05-17T05:39:58.838+0100 F - [thread42] Got signal: 6 (Aborted).
      0x558f0e2aa901 0x558f0e2a9b19 0x558f0e2a9ffd 0x7fc149cc8100 0x7fc14992c5f7 0x7fc14992dce8 0x558f0ca0b06e 0x558f0dbe166f 0x558f0dbd2c50 0x558f0d15d1d7 0x558f0db8094d 0x558f0db81f78 0x558f0d0f5db2 0x558f0dd7f0aa 0x558f0dd7fa6c 0x558f0dd7fe44 0x558f0ddf86c9 0x558f0ddf8911 0x558f0d0f4cbd 0x558f0e3ba040 0x7fc149cc0dc5 0x7fc1499edc9d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"558F0C07C000","o":"222E901","s":"_ZN5mongo15printStackTraceERSo"},{"b":"558F0C07C000","o":"222DB19"},{"b":"558F0C07C000","o":"222DFFD"},{"b":"7FC149CB9000","o":"F100"},{"b":"7FC1498F7000","o":"355F7","s":"gsignal"},{"b":"7FC1498F7000","o":"36CE8","s":"abort"},{"b":"558F0C07C000","o":"98F06E","s":"_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j"},{"b":"558F0C07C000","o":"1B6566F","s":"_ZN5mongo10LockerImplILb1EE15lockMMAPV1FlushEv"},{"b":"558F0C07C000","o":"1B56C50","s":"_ZN5mongo4Lock6DBLockC2EPNS_16OperationContextENS_10StringDataENS_8LockModeE"},{"b":"558F0C07C000","o":"10E11D7","s":"_ZN5mongo20SessionsCollectionRS23setupSessionsCollectionEPNS_16OperationContextE"},{"b":"558F0C07C000","o":"1B0494D","s":"_ZN5mongo23LogicalSessionCacheImpl8_refreshEPNS_6ClientE"},{"b":"558F0C07C000","o":"1B05F78","s":"_ZN5mongo23LogicalSessionCacheImpl16_periodicRefreshEPNS_6ClientE"},{"b":"558F0C07C000","o":"1079DB2"},{"b":"558F0C07C000","o":"1D030AA","s":"_ZN4asio6detail14strand_service8dispatchINS0_7binder1ISt8functionIFvSt10error_codeEES5_EEEEvRPNS1_11strand_implERT_"},{"b":"558F0C07C000","o":"1D03A6C","s":"_ZN4asio6detail14strand_service8dispatchINS0_17rewrapped_handlerINS0_7binder1INS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEES9_EESB_EEEEvRPNS1_11strand_implERT_"},{"b":"558F0C07C000","o":"1D03E44","s":"_ZN4asio6detail12wait_handlerINS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEEE11do_completeEPvPNS0_19scheduler_operationERKS6_m"},{"b":"558F0C07C000","o":"1D7C6C9","s":"_ZN4asio6detail9scheduler10do_run_oneERNS0_27conditionally_enabled_mutex11scoped_lockERNS0_21scheduler_thread_infoERKSt10error_code"},{"b":"558F0C07C000","o":"1D7C911","s":"_ZN4asio6detail9scheduler3runERSt10error_code"},{"b":"558F0C07C000","o":"1078CBD"},{"b":"558F0C07C000","o":"233E040"},{"b":"7FC149CB9000","o":"7DC5"},{"b":"7FC1498F7000","o":"F6C9D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.6.4", "gitVersion" : "d0181a711f7e7f39e60b5aeb1dc7097bf6ae5856", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "4.4.41-36.55.amzn1.x86_64", "version" : "#1 SMP Wed Jan 18 01:03:26 UTC 2017", "machine" : "x86_64" }, "somap" : [ { "b" : "558F0C07C000", "elfType" : 3, "buildId" : "A1E0876F649FBE5569FAEF264DA851938158F2BC" }, { "b" : "7FFF84DC7000", "elfType" : 3, "buildId" : "F3A72C9C20A0FD0E902360D4BC280F2002571040" }, { "b" : "7FC14AE4C000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "1285F9516FFCF13FC00BD135C5634AF2EB16C80B" }, { "b" : "7FC14ABDF000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "6AF827B6FD200DFDFE70B2BC8D66BBC9881E8817" }, { "b" : "7FC14A7F9000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "83F15DBCD0653F417E98354BC1EED6F96A758367" }, { "b" : "7FC14A5F1000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "2B3151901240D9E854E18E6D0B181C4D580ABA9C" }, { "b" : "7FC14A3ED000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "6335077ACD51527BE9F2F18451A88E2B7350C5B6" }, { "b" : "7FC14A0EB000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "6E343508D15886FE83C438DF4560CE40BEB64B56" }, { "b" : "7FC149ED5000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "3FD5F89DE59E124AB1419A0BD16775B4096E84FD" }, { "b" : "7FC149CB9000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "0836319AA81CDFE97DA2666963F62DE6A2A61346" }, { "b" : "7FC1498F7000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "5D38A77E8D79E98D717281031C39B9A341323BD1" }, { "b" : "7FC14B066000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "93D931BA041229929E5F099514B20E36A70BD651" }, { "b" : "7FC1496AB000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "E203354E7F907ACC8C3028FE465541B666DCFBA0" }, { "b" : "7FC1493C6000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "D769C8FFAF8772FDA55031ABF2E167DF2207E378" }, { "b" : "7FC1491C3000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FC148F91000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "6C2243D37143F7FD1E16ED1F6CE4D7F16C2D7EF1" }, { "b" : "7FC148D7B000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FC148B6C000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "744272FEAAABCE629AB9E11FAA4A96AEBE8BC2B4" }, { "b" : "7FC148969000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FC148748000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "F5054DC94443326819FBF3065CFDF5E4726F57EE" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x558f0e2aa901]
       mongod(+0x222DB19) [0x558f0e2a9b19]
       mongod(+0x222DFFD) [0x558f0e2a9ffd]
       libpthread.so.0(+0xF100) [0x7fc149cc8100]
       libc.so.6(gsignal+0x37) [0x7fc14992c5f7]
       libc.so.6(abort+0x148) [0x7fc14992dce8]
       mongod(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0x0) [0x558f0ca0b06e]
       mongod(_ZN5mongo10LockerImplILb1EE15lockMMAPV1FlushEv+0x4F) [0x558f0dbe166f]
       mongod(_ZN5mongo4Lock6DBLockC2EPNS_16OperationContextENS_10StringDataENS_8LockModeE+0x90) [0x558f0dbd2c50]
       mongod(_ZN5mongo20SessionsCollectionRS23setupSessionsCollectionEPNS_16OperationContextE+0x77) [0x558f0d15d1d7]
       mongod(_ZN5mongo23LogicalSessionCacheImpl8_refreshEPNS_6ClientE+0xCD) [0x558f0db8094d]
       mongod(_ZN5mongo23LogicalSessionCacheImpl16_periodicRefreshEPNS_6ClientE+0x28) [0x558f0db81f78]
       mongod(+0x1079DB2) [0x558f0d0f5db2]
       mongod(_ZN4asio6detail14strand_service8dispatchINS0_7binder1ISt8functionIFvSt10error_codeEES5_EEEEvRPNS1_11strand_implERT_+0x7A) [0x558f0dd7f0aa]
       mongod(_ZN4asio6detail14strand_service8dispatchINS0_17rewrapped_handlerINS0_7binder1INS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEES9_EESB_EEEEvRPNS1_11strand_implERT_+0x3AC) [0x558f0dd7fa6c]
       mongod(_ZN4asio6detail12wait_handlerINS0_15wrapped_handlerINS_10io_context6strandESt8functionIFvSt10error_codeEENS0_26is_continuation_if_runningEEEE11do_completeEPvPNS0_19scheduler_operationERKS6_m+0x164) [0x558f0dd7fe44]
       mongod(_ZN4asio6detail9scheduler10do_run_oneERNS0_27conditionally_enabled_mutex11scoped_lockERNS0_21scheduler_thread_infoERKSt10error_code+0x389) [0x558f0ddf86c9]
       mongod(_ZN4asio6detail9scheduler3runERSt10error_code+0xD1) [0x558f0ddf8911]
       mongod(+0x1078CBD) [0x558f0d0f4cbd]
       mongod(+0x233E040) [0x558f0e3ba040]
       libpthread.so.0(+0x7DC5) [0x7fc149cc0dc5]
       libc.so.6(clone+0x6D) [0x7fc1499edc9d]
      ----- END BACKTRACE -----
      

            Assignee:
            daniel.hatcher@mongodb.com Danny Hatcher (Inactive)
            Reporter:
            chriswatton Christopher Watton
            Votes:
            0 Vote for this issue
            Watchers:
            9 Start watching this issue

              Created:
              Updated:
              Resolved: