Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-24711

ASIO connections that have already timed out before setup should not error

    • Fully Compatible
    • ALL
    • Hide

      It happened by itself, probably network segmentation.

      Show
      It happened by itself, probably network segmentation.
    • Platforms 16 (06/24/16), Platforms 17 (07/15/16)
    • 0

      I have a multi site setup of MongoDB 3.2.7 in four locations, today, two config servers crashed with the following log.

      016-06-21T18:46:17.059+0400 I REPL     [ReplicationExecutor] Error in heartbeat request to mcfg01:27019; ExceededTimeLimit: Couldn't get a connection within the time limit
      2016-06-21T18:46:20.447+0400 I ACCESS   [conn1] Successfully authenticated as principal root on admin
      2016-06-21T18:46:27.059+0400 I -        [NetworkInterfaceASIO-Replication-0] Invariant failure _connection.is_initialized() src/mongo/executor/network_interface_asio_operation.cpp 142
      2016-06-21T18:46:27.059+0400 I -        [NetworkInterfaceASIO-Replication-0] 
      
      ***aborting after invariant() failure
      
      
      2016-06-21T18:46:27.421+0400 F -        [NetworkInterfaceASIO-Replication-0] Got signal: 6 (Aborted).
      
       0x1319fa2 0x13190f9 0x1319902 0x7f537c2a8340 0x7f537bf09cc9 0x7f537bf0d0d8 0x12a380b 0x10e766d 0x10c6643 0x10c763c 0x10c7ba7 0x10c82d9 0x1335cb1 0x1335ed1 0x133a06f 0x10d36b5 0x1b34160 0x7f537c2a0182 0x7f537bfcd47d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"F19FA2","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"F190F9"},{"b":"400000","o":"F19902"},{"b":"7F537C298000","o":"10340"},{"b":"7F537BED3000","o":"36CC9","s":"gsignal"},{"b":"7F537BED3000","o":"3A0D8","s":"abort"},{"b":"400000","o":"EA380B","s":"_ZN5mongo15invariantFailedEPKcS1_j"},{"b":"400000","o":"CE766D"},{"b":"400000","o":"CC6643"},{"b":"400000","o":"CC763C"},{"b":"400000","o":"CC7BA7"},{"b":"400000","o":"CC82D9"},{"b":"400000","o":"F35CB1","s":"_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code"},{"b":"400000","o":"F35ED1","s":"_ZN4asio6detail9scheduler3runERSt10error_code"},{"b":"400000","o":"F3A06F","s":"_ZN4asio10io_service3runEv"},{"b":"400000","o":"CD36B5"},{"b":"400000","o":"1734160","s":"execute_native_thread_routine"},{"b":"7F537C298000","o":"8182"},{"b":"7F537BED3000","o":"FA47D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.2.7", "gitVersion" : "4249c1d2b5999ebbf1fdf3bc0e0e3b3ff5c0aaf2", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.16.0-60-generic", "version" : "#80~14.04.1-Ubuntu SMP Wed Jan 20 13:37:48 UTC 2016", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "A8DC02B241532EEAD31FFE2633EE8C8E565C1F1A" }, { "b" : "7FFE735E9000", "elfType" : 3, "buildId" : "E909F3BEA8B99B483153329FF4F90B288A068655" }, { "b" : "7F537D1B9000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "D08DD65F97859C71BB2CBBF1043BD968EFE18AAD" }, { "b" : "7F537CDDE000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "F86FA9FB4ECEB4E06B40DBDF761A4172B70A4229" }, { "b" : "7F537CBD6000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "B376100CAB1EAC4E5DE066EACFC282BF7C0B54F3" }, { "b" : "7F537C9D2000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "67699FFDA9FD2A552032E0652A242E82D65AA10D" }, { "b" : "7F537C6CC000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "EF3F6DFFA1FBE48436EC6F45CD3AABA157064BB4" }, { "b" : "7F537C4B6000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "36311B4457710AE5578C4BF00791DED7359DBB92" }, { "b" : "7F537C298000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "AF06068681750736E0524DF17D5A86CB2C3F765C" }, { "b" : "7F537BED3000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "5382058B69031CAA9B9996C11061CD164C9398FF" }, { "b" : "7F537D418000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "2A816C3EBBA4E12813FBD34B06FBD25BC892A67F" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x1319fa2]
       mongod(+0xF190F9) [0x13190f9]
       mongod(+0xF19902) [0x1319902]
       libpthread.so.0(+0x10340) [0x7f537c2a8340]
       libc.so.6(gsignal+0x39) [0x7f537bf09cc9]
       libc.so.6(abort+0x148) [0x7f537bf0d0d8]
       mongod(_ZN5mongo15invariantFailedEPKcS1_j+0xCB) [0x12a380b]
       mongod(+0xCE766D) [0x10e766d]
       mongod(+0xCC6643) [0x10c6643]
       mongod(+0xCC763C) [0x10c763c]
       mongod(+0xCC7BA7) [0x10c7ba7]
       mongod(+0xCC82D9) [0x10c82d9]
       mongod(_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code+0x2F1) [0x1335cb1]
       mongod(_ZN4asio6detail9scheduler3runERSt10error_code+0xC1) [0x1335ed1]
       mongod(_ZN4asio10io_service3runEv+0x2F) [0x133a06f]
       mongod(+0xCD36B5) [0x10d36b5]
       mongod(execute_native_thread_routine+0x20) [0x1b34160]
       libpthread.so.0(+0x8182) [0x7f537c2a0182]
       libc.so.6(clone+0x6D) [0x7f537bfcd47d]
      -----  END BACKTRACE  -----
      

            Votes:
            1 Vote for this issue
            Watchers:
            21 Start watching this issue

              Created:
              Updated:
              Resolved: