Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-24969

Abort on loss of network in replicaset

    XMLWordPrintable

    Details

    • Type: Bug
    • Status: Closed
    • Priority: Critical - P2
    • Resolution: Duplicate
    • Affects Version/s: 3.2.6
    • Fix Version/s: None
    • Component/s: Replication
    • Labels:
      None
    • Operating System:
      ALL

      Description

      When rebooting a firewall this morning, every single member of the replicaset crashed:

      2016-07-09T11:32:50.121-0500 I REPL     [ReplicationExecutor] Error in heartbeat request to mongo003.ev.onspring.us:27017; ExceededTimeLimit: Couldn't get a connection within the time limit
      2016-07-09T11:32:53.120-0500 I -        [NetworkInterfaceASIO-Replication-0] Invariant failure _connection.is_initialized() src/mongo/executor/network_interface_asio_operation.cpp 142
      2016-07-09T11:32:53.120-0500 I -        [NetworkInterfaceASIO-Replication-0]
       
      ***aborting after invariant() failure
       
       
      2016-07-09T11:32:53.150-0500 F -        [NetworkInterfaceASIO-Replication-0] Got signal: 6 (Aborted).
       
       0x131dad2 0x131cc09 0x131d412 0x7f6ba706d8d0 0x7f6ba6ce8067 0x7f6ba6ce9448 0x12a6ffb 0x10ea3dd 0x10c92e3 0x10ca2dc 0x10ca847 0x10caf79 0x1339861 0x1339a81 0x133dc1f 0x10d63b5 0x1b383b0 0x7f6ba70660a4 0x7f6ba6d9b87d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"F1DAD2","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"F1CC09"},{"b":"400000","o":"F1D412"},{"b":"7F6BA705E000","o":"F8D0"},{"b":"7F6BA6CB3000","o":"35067","s":"gsignal"},{"b":"7F6BA6CB3000","o":"36448","s":"abort"},{"b":"400000","o":"EA6FFB","s":"_ZN5mongo15invariantFailedEPKcS1_j"},{"b":"400000","o":"CEA3DD"},{"b":"400000","o":"CC92E3"},{"b":"400000","o":"CCA2DC"},{"b":"400000","o":"CCA847"},{"b":"400000","o":"CCAF79"},{"b":"400000","o":"F39861","s":"_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code"},{"b":"400000","o":"F39A81","s":"_ZN4asio6detail9scheduler3runERSt10error_code"},{"b":"400000","o":"F3DC1F","s":"_ZN4asio10io_service3runEv"},{"b":"400000","o":"CD63B5"},{"b":"400000","o":"17383B0","s":"execute_native_thread_routine"},{"b":"7F6BA705E000","o":"80A4"},{"b":"7F6BA6CB3000","o":"E887D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.2.7", "gitVersion" : "4249c1d2b5999ebbf1fdf3bc0e0e3b3ff5c0aaf2", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.16.0-4-amd64", "version" : "#1 SMP Debian 3.16.7-ckt20-1+deb8u4 (2016-02-29)", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "C77A8E4B49470F8A1315347C44FD6F7FCFA6C4B7" }, { "b" : "7FFE287CB000", "path" : "linux-vdso.so.1", "elfType" : 3, "buildId" : "22542662ACC65CD6DEB369E37E05EE9F0B180F68" }, { "b" : "7F6BA7F9A000", "path" : "/usr/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "C72A89930DB5C11FE5737AA6D0CA245AA4C08329" }, { "b" : "7F6BA7B9E000", "path" : "/usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "C72426F83BD6D6E49DF4B366AA39FAD5837C8C1F" }, { "b" : "7F6BA7996000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "C96926159E07B6128EFEC30001EF8AD3D24672DD" }, { "b" : "7F6BA7792000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "1FDE060F62834E445210B73CD25C7ABDF905B542" }, { "b" : "7F6BA7491000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "53024C3BBDBA1F0CBCC01D6A80FA2CBDFE33EB42" }, { "b" : "7F6BA727B000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "D5FB04F64B3DAEA6D6B68B5E8B9D4D2BC1A6E1FC" }, { "b" : "7F6BA705E000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "424D19435BCBB4D6E67DC19FD6EA95AC08B09237" }, { "b" : "7F6BA6CB3000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "2BBA9D358BAC63FF81597767B5BC229316BD12E0" }, { "b" : "7F6BA81FB000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "486275655F9C95F0B45F9E4CEA1A448DB59625D8" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x131dad2]
       mongod(+0xF1CC09) [0x131cc09]
       mongod(+0xF1D412) [0x131d412]
       libpthread.so.0(+0xF8D0) [0x7f6ba706d8d0]
       libc.so.6(gsignal+0x37) [0x7f6ba6ce8067]
       libc.so.6(abort+0x148) [0x7f6ba6ce9448]
       mongod(_ZN5mongo15invariantFailedEPKcS1_j+0xCB) [0x12a6ffb]
       mongod(+0xCEA3DD) [0x10ea3dd]
       mongod(+0xCC92E3) [0x10c92e3]
       mongod(+0xCCA2DC) [0x10ca2dc]
       mongod(+0xCCA847) [0x10ca847]
       mongod(+0xCCAF79) [0x10caf79]
       mongod(_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code+0x2F1) [0x1339861]
       mongod(_ZN4asio6detail9scheduler3runERSt10error_code+0xC1) [0x1339a81]
       mongod(_ZN4asio10io_service3runEv+0x2F) [0x133dc1f]
       mongod(+0xCD63B5) [0x10d63b5]
       mongod(execute_native_thread_routine+0x20) [0x1b383b0]
       libpthread.so.0(+0x80A4) [0x7f6ba70660a4]
       libc.so.6(clone+0x6D) [0x7f6ba6d9b87d]
      -----  END BACKTRACE  -----
      

        Attachments

          Issue Links

            Activity

              People

              • Votes:
                0 Vote for this issue
                Watchers:
                4 Start watching this issue

                Dates

                • Created:
                  Updated:
                  Resolved: