Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-27863

Reschedule early alarms in NetworkInterfaceASIO to avoid mongos crash

    Details

    • Type: Bug
    • Status: Closed
    • Priority: Critical - P2
    • Resolution: Fixed
    • Affects Version/s: 3.4.1
    • Fix Version/s: 3.4.3, 3.5.4
    • Component/s: Shell
    • Labels:
      None
    • Backwards Compatibility:
      Fully Compatible
    • Operating System:
      ALL
    • Backport Requested:
      v3.4
    • Steps To Reproduce:
      Hide

      It happens in non busy hours

      Show
      It happens in non busy hours

      Description

      Mongos servers are crashing with the following log messages

      2017-01-31T05:16:32.309+0400 I -        [ReplicaSetMonitor-TaskExecutor-0] Invariant failure now() >= when src/mongo/executor/thread_pool_task_executor.cpp 294
      2017-01-31T05:16:32.309+0400 I -        [ReplicaSetMonitor-TaskExecutor-0] 
       
      ***aborting after invariant() failure
       
       
      2017-01-31T05:16:32.349+0400 F -        [ReplicaSetMonitor-TaskExecutor-0] Got signal: 6 (Aborted).
       
       0x7f8bca5635b1 0x7f8bca5626a9 0x7f8bca562b8d 0x7f8bc8629330 0x7f8bc828ac37 0x7f8bc828e028 0x7f8bc9d9f204 0x7f8bca2b8f18 0x7f8bca290225 0x7f8bca641164 0x7f8bca641361 0x7f8bca28f32f 0x7f8bca941ba0 0x7f8bc8621184 0x7f8bc834e37d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"7F8BC99BF000","o":"BA45B1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"7F8BC99BF000","o":"BA36A9"},{"b":"7F8BC99BF000","o":"BA3B8D"},{"b":"7F8BC8619000","o":"10330"},{"b":"7F8BC8254000","o":"36C37","s":"gsignal"},{"b":"7F8BC8254000","o":"3A028","s":"abort"},{"b":"7F8BC99BF000","o":"3E0204","s":"_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j"},{"b":"7F8BC99BF000","o":"8F9F18"},{"b":"7F8BC99BF000","o":"8D1225"},{"b":"7F8BC99BF000","o":"C82164","s":"_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code"},{"b":"7F8BC99BF000","o":"C82361","s":"_ZN4asio6detail9scheduler3runERSt10error_code"},{"b":"7F8BC99BF000","o":"8D032F"},{"b":"7F8BC99BF000","o":"F82BA0"},{"b":"7F8BC8619000","o":"8184"},{"b":"7F8BC8254000","o":"FA37D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.4.1", "gitVersion" : "5e103c4f5583e2566a45d740225dc250baacfbd7", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.19.0-33-generic", "version" : "#38~14.04.1-Ubuntu SMP Fri Nov 6 18:17:28 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "b" : "7F8BC99BF000", "elfType" : 3, "buildId" : "0E781D7641A3474BB32487BBA122619FE8606F8F" }, { "b" : "7FFCC0BE6000", "elfType" : 3, "buildId" : "3AA714083E1D7D719D6C5A7B1A4A8D3649727CB8" }, { "b" : "7F8BC953B000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "15A6A69B862FAC0AA589921DB1B0DF8A673FEC81" }, { "b" : "7F8BC915F000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "607C4A6F3428B6F065D55691AFDC4E343786A8A5" }, { "b" : "7F8BC8F57000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "E2A6DD5048A0A051FD61043BDB69D8CC68192AB7" }, { "b" : "7F8BC8D53000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "DA9B8C234D0FE9FD8CAAC8970A7EC1B6C8F6623F" }, { "b" : "7F8BC8A4D000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "D144258E614900B255A31F3FD2283A878670D5BC" }, { "b" : "7F8BC8837000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "36311B4457710AE5578C4BF00791DED7359DBB92" }, { "b" : "7F8BC8619000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "31E9F21AE8C10396171F1E13DA15780986FA696C" }, { "b" : "7F8BC8254000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "CF699A15CAAE64F50311FC4655B86DC39A479789" }, { "b" : "7F8BC979A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "D0F537904076D73F29E4A37341F8A449E2EF6CD0" }, { "b" : "7F8BC8049000", "path" : "/lib/x86_64-linux-gnu/libnss_files.so.2", "elfType" : 3, "buildId" : "A0BDDA36D5AD1390D4DF158D7AD7CC866E534C28" }, { "b" : "7F8BC7E43000", "path" : "/lib/x86_64-linux-gnu/libnss_dns.so.2", "elfType" : 3, "buildId" : "A5721A2FACA0CB423F6C851DF8B7A96C52E94691" }, { "b" : "7F8BC7C28000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "FE705AA7E3872B43EA9E99FAF1897F7F85FC0AC9" } ] }}
       mongos(_ZN5mongo15printStackTraceERSo+0x41) [0x7f8bca5635b1]
       mongos(+0xBA36A9) [0x7f8bca5626a9]
       mongos(+0xBA3B8D) [0x7f8bca562b8d]
       libpthread.so.0(+0x10330) [0x7f8bc8629330]
       libc.so.6(gsignal+0x37) [0x7f8bc828ac37]
       libc.so.6(abort+0x148) [0x7f8bc828e028]
       mongos(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0x0) [0x7f8bc9d9f204]
       mongos(+0x8F9F18) [0x7f8bca2b8f18]
       mongos(+0x8D1225) [0x7f8bca290225]
       mongos(_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code+0x324) [0x7f8bca641164]
       mongos(_ZN4asio6detail9scheduler3runERSt10error_code+0xD1) [0x7f8bca641361]
       mongos(+0x8D032F) [0x7f8bca28f32f]
       mongos(+0xF82BA0) [0x7f8bca941ba0]
       libpthread.so.0(+0x8184) [0x7f8bc8621184]
       libc.so.6(clone+0x6D) [0x7f8bc834e37d]
      -----  END BACKTRACE  -----
      

        Attachments

          Issue Links

            Activity

              People

              • Votes:
                0 Vote for this issue
                Watchers:
                11 Start watching this issue

                Dates

                • Created:
                  Updated:
                  Resolved: