Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-23686

auto2.js - fatal assertion 28735 ShutdownInProgress

    • Type: Icon: Bug Bug
    • Resolution: Done
    • Priority: Icon: Major - P3 Major - P3
    • 3.2.8, 3.3.5
    • Affects Version/s: None
    • Component/s: None
    • Labels:
    • Fully Compatible
    • ALL
    • Sharding 12 (04/01/16), Sharding 13 (04/22/16)
    • 0

      sharding_WT failed on ubuntu1410-fast-clang

      auto2.js - Logs | History

      Encountered an fassert during shutdown:

      js_test:auto2] 2016-03-05T06:16:48.703+0000 s20265| 2016-03-05T06:16:48.680+0000 I -        [NetworkInterfaceASIO-TaskExecutorPool-1-0] Fatal assertion 28735 ShutdownInProgress: Shutdown in progress
      [js_test:auto2] 2016-03-05T06:16:48.703+0000 s20265| 2016-03-05T06:16:48.680+0000 I -        [NetworkInterfaceASIO-TaskExecutorPool-1-0]
      [js_test:auto2] 2016-03-05T06:16:48.703+0000 s20265|
      [js_test:auto2] 2016-03-05T06:16:48.703+0000 s20265| ***aborting after fassert() failure
      [js_test:auto2] 2016-03-05T06:16:48.703+0000 s20265|
      [js_test:auto2] 2016-03-05T06:16:48.703+0000 s20265|
      [js_test:auto2] 2016-03-05T06:16:48.703+0000 d20261| 2016-03-05T06:16:48.680+0000 I NETWORK  [conn5] end connection 10.63.166.37:46780 (24 connections now open)
      [js_test:auto2] 2016-03-05T06:16:48.703+0000 s20265| 2016-03-05T06:16:48.682+0000 F -        [NetworkInterfaceASIO-TaskExecutorPool-1-0] Got signal: 6 (Aborted).
      [js_test:auto2] 2016-03-05T06:16:48.704+0000 s20265|
      [js_test:auto2] 2016-03-05T06:16:48.704+0000 s20265|  0xbd6b3b 0xbd5f9d 0x7f741d332c90 0x7f741cf95e37 0x7f741cf97528 0xb55252 0x9d6ac8 0x9d7965 0x9cc473 0x9c0994 0x9c1bff 0x9c6337 0x9c6003 0x99aca8 0x99aa28 0x9a38c3 0x9a40f2 0x9a3fad 0x9a3dc3 0x9a3c39 0xbec3a2 0xbe3cb4 0xbe3b6b 0x9b8da4 0x7f741d32b0a5 0x7f741d058cfd
      [js_test:auto2] 2016-03-05T06:16:48.704+0000 s20265| ----- BEGIN BACKTRACE -----
      [js_test:auto2] 2016-03-05T06:16:48.706+0000 s20265| {"backtrace":[{"b":"400000","o":"7D6B3B","s":"_ZN5mongo15printStackTraceERNSt3__113basic_ostreamIcNS0_11char_traitsIcEEEE"},{"b":"400000","o":"7D5F9D"},{"b":"7F741D323000","o":"FC90"},{"b":"7F741CF5F000","o":"36E37","s":"gsignal"},{"b":"7F741CF5F000","o":"38528","s":"abort"},{"b":"400000","o":"755252","s":"_ZN5mongo23fassertFailedWithStatusEiRKNS_6StatusE"},{"b":"400000","o":"5D6AC8","s":"_ZN5mongo8executor22ThreadPoolTaskExecutor23scheduleIntoPool_inlockEPNSt3__14listINS2_10shared_ptrINS1_13CallbackStateEEENS2_9allocatorIS6_EEEERKNS2_15__list_iteratorIS6_PvEESF_NS2_11unique_lockINS2_5mutexEEE"},{"b":"400000","o":"5D7965"},{"b":"400000","o":"5CC473","s":"_ZN5mongo8executor20NetworkInterfaceASIO7AsyncOp6finishERKNS_10StatusWithINS0_21RemoteCommandResponseEEE"},{"b":"400000","o":"5C0994","s":"_ZN5mongo8executor20NetworkInterfaceASIO18_completeOperationEPNS1_7AsyncOpERKNS_10StatusWithINS0_21RemoteCommandResponseEEE"},{"b":"400000","o":"5C1BFF","s":"_ZN5mongo8executor20NetworkInterfaceASIO20_completedOpCallbackEPNS1_7AsyncOpE"},{"b":"400000","o":"5C6337"},{"b":"400000","o":"5C6003"},{"b":"400000","o":"59ACA8","s":"_ZN4asio19asio_handler_invokeINS_6detail7binder2IRNSt3__18functionIFvNS3_10error_codeEmEEES5_mEEEEvRT_z"},{"b":"400000","o":"59AA28","s":"_ZN4asio6detail14strand_service8dispatchINS0_7binder2IRNSt3__18functionIFvNS4_10error_codeEmEEES6_mEEEEvRPNS1_11strand_implERT_"},{"b":"400000","o":"5A38C3","s":"_ZN4asio6detail7read_opINS_19basic_stream_socketINS_2ip3tcpENS_21stream_socket_serviceIS4_EEEENS_17mutable_buffers_1ENS0_14transfer_all_tENS0_15wrapped_handlerINS_10io_service6strandENSt3__18functionIFvNSD_10error_codeEmEEENS0_26is_continuation_if_runningEEEEclERKSF_mi"},{"b":"400000","o":"5A40F2","s":"_ZN4asio6detail18completion_handlerINS0_17rewrapped_handlerINS0_7binder2INS0_7read_opINS_19basic_stream_socketINS_2ip3tcpENS_21stream_socket_serviceIS7_EEEENS_17mutable_buffers_1ENS0_14transfer_all_tENS0_15wrapped_handlerINS_10io_service6strandENSt3__18functionIFvNSG_10error_codeEmEEENS0_26is_continuation_if_runningEEEEESI_mEESK_EEE11do_completeEPvPNS0_19scheduler_operationERKSI_m"},{"b":"400000","o":"5A3FAD","s":"_ZN4asio6detail14strand_service8dispatchINS0_17rewrapped_handlerINS0_7binder2INS0_7read_opINS_19basic_stream_socketINS_2ip3tcpENS_21stream_socket_serviceIS8_EEEENS_17mutable_buffers_1ENS0_14transfer_all_tENS0_15wrapped_handlerINS_10io_service6strandENSt3__18functionIFvNSH_10error_codeEmEEENS0_26is_continuation_if_runningEEEEESJ_mEESL_EEEEvRPNS1_11strand_implERT_"},{"b":"400000","o":"5A3DC3","s":"_ZN4asio6detail19asio_handler_invokeINS0_7binder2INS0_7read_opINS_19basic_stream_socketINS_2ip3tcpENS_21stream_socket_serviceIS6_EEEENS_17mutable_buffers_1ENS0_14transfer_all_tENS0_15wrapped_handlerINS_10io_service6strandENSt3__18functionIFvNSF_10error_codeEmEEENS0_26is_continuation_if_runningEEEEESH_mEESE_SJ_SK_EEvRT_PNSC_IT0_T1_T2_EE"},{"b":"400000","o":"5A3C39","s":"_ZN4asio6detail23reactive_socket_recv_opINS_17mutable_buffers_1ENS0_7read_opINS_19basic_stream_socketINS_2ip3tcpENS_21stream_socket_serviceIS6_EEEES2_NS0_14transfer_all_tENS0_15wrapped_handlerINS_10io_service6strandENSt3__18functionIFvNSE_10error_codeEmEEENS0_26is_continuation_if_runningEEEEEE11do_completeEPvPNS0_19scheduler_operationERKSG_m"},{"b":"400000","o":"7EC3A2","s":"_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKNSt3__110error_codeE"},{"b":"400000","o":"7E3CB4","s":"_ZN4asio6detail9scheduler3runERNSt3__110error_codeE"},{"b":"400000","o":"7E3B6B","s":"_ZN4asio10io_service3runEv"},{"b":"400000","o":"5B8DA4"},{"b":"7F741D323000","o":"80A5"},{"b":"7F741CF5F000","o":"F9CFD","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.3.2-122-g7652290", "gitVersion" : "76522903744fba9a660c2d2b9db61507578916b3", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.16.0-25-generic", "version" : "#33-Ubuntu SMP Tue Nov 4 12:06:54 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000" }, { "b" : "7FFF50FFC000", "elfType" : 3, "buildId" : "B43B3054EC49983B9ECFC6BD437E70C564D0AE41" }, { "b" : "7F741E4A9000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "4E96203F4FE17D3446F48226AAEA8DA6DEA8FFD0" }, { "b" : "7F741E248000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "44ADE6AD24BE7656EDC5B05A53974EA73F35D423" }, { "b" : "7F741DE65000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "013EF089B82F17AD7F2BDB94FEC56724D84D7A17" }, { "b" : "7F741DC5D000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "54EF3A97A3E71418DD088B40AF51A00457834A17" }, { "b" : "7F741DA59000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "794CD87201C9778112E22BF5E2C0FBFB3390D29F" }, { "b" : "7F741D758000", "path" : "/usr/lib/x86_64-linux-gnu/libc++.so.1", "elfType" : 3 }, { "b" : "7F741D541000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "7C6E98219378EBD1AA0D4CD671E8FF1589C04C4A" }, { "b" : "7F741D323000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "921196598AF41AFF8DE42EEFB8561243610F34C3" }, { "b" : "7F741CF5F000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "95287BE8ACCCC7B5723F4306E6A5ECA6DFE7BFFD" }, { "b" : "7F741E7AF000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "9240DBBD1DB14E756141EEE1FDDB67D3B77864E7" } ] }}
      [js_test:auto2] 2016-03-05T06:16:48.706+0000 s20265|  mongos(_ZN5mongo15printStackTraceERNSt3__113basic_ostreamIcNS0_11char_traitsIcEEEE+0x2B) [0xbd6b3b]
      [js_test:auto2] 2016-03-05T06:16:48.706+0000 s20265|  mongos(+0x7D5F9D) [0xbd5f9d]
      [js_test:auto2] 2016-03-05T06:16:48.706+0000 s20265|  libpthread.so.0(+0xFC90) [0x7f741d332c90]
      [js_test:auto2] 2016-03-05T06:16:48.706+0000 s20265|  libc.so.6(gsignal+0x37) [0x7f741cf95e37]
      [js_test:auto2] 2016-03-05T06:16:48.706+0000 s20265|  libc.so.6(abort+0x148) [0x7f741cf97528]
      [js_test:auto2] 2016-03-05T06:16:48.707+0000 s20265|  mongos(_ZN5mongo23fassertFailedWithStatusEiRKNS_6StatusE+0x322) [0xb55252]
      [js_test:auto2] 2016-03-05T06:16:48.707+0000 s20265|  mongos(_ZN5mongo8executor22ThreadPoolTaskExecutor23scheduleIntoPool_inlockEPNSt3__14listINS2_10shared_ptrINS1_13CallbackStateEEENS2_9allocatorIS6_EEEERKNS2_15__list_iteratorIS6_PvEESF_NS2_11unique_lockINS2_5mutexEEE+0x208) [0x9d6ac8]
      [js_test:auto2] 2016-03-05T06:16:48.707+0000 s20265|  mongos(+0x5D7965) [0x9d7965]
      [js_test:auto2] 2016-03-05T06:16:48.707+0000 s20265|  mongos(_ZN5mongo8executor20NetworkInterfaceASIO7AsyncOp6finishERKNS_10StatusWithINS0_21RemoteCommandResponseEEE+0x253) [0x9cc473]
      [js_test:auto2] 2016-03-05T06:16:48.707+0000 s20265|  mongos(_ZN5mongo8executor20NetworkInterfaceASIO18_completeOperationEPNS1_7AsyncOpERKNS_10StatusWithINS0_21RemoteCommandResponseEEE+0x484) [0x9c0994]
      [js_test:auto2] 2016-03-05T06:16:48.707+0000 s20265|  mongos(_ZN5mongo8executor20NetworkInterfaceASIO20_completedOpCallbackEPNS1_7AsyncOpE+0x5F) [0x9c1bff]
      [js_test:auto2] 2016-03-05T06:16:48.707+0000 s20265|  mongos(+0x5C6337) [0x9c6337]
      [js_test:auto2] 2016-03-05T06:16:48.707+0000 s20265|  mongos(+0x5C6003) [0x9c6003]
      [js_test:auto2] 2016-03-05T06:16:48.707+0000 s20265|  mongos(_ZN4asio19asio_handler_invokeINS_6detail7binder2IRNSt3__18functionIFvNS3_10error_codeEmEEES5_mEEEEvRT_z+0x88) [0x99aca8]
      [js_test:auto2] 2016-03-05T06:16:48.708+0000 s20265|  mongos(_ZN4asio6detail14strand_service8dispatchINS0_7binder2IRNSt3__18functionIFvNS4_10error_codeEmEEES6_mEEEEvRPNS1_11strand_implERT_+0x68) [0x99aa28]
      [js_test:auto2] 2016-03-05T06:16:48.708+0000 s20265|  mongos(_ZN4asio6detail7read_opINS_19basic_stream_socketINS_2ip3tcpENS_21stream_socket_serviceIS4_EEEENS_17mutable_buffers_1ENS0_14transfer_all_tENS0_15wrapped_handlerINS_10io_service6strandENSt3__18functionIFvNSD_10error_codeEmEEENS0_26is_continuation_if_runningEEEEclERKSF_mi+0x83) [0x9a38c3]
      [js_test:auto2] 2016-03-05T06:16:48.708+0000 s20265|  mongos(_ZN4asio6detail18completion_handlerINS0_17rewrapped_handlerINS0_7binder2INS0_7read_opINS_19basic_stream_socketINS_2ip3tcpENS_21stream_socket_serviceIS7_EEEENS_17mutable_buffers_1ENS0_14transfer_all_tENS0_15wrapped_handlerINS_10io_service6strandENSt3__18functionIFvNSG_10error_codeEmEEENS0_26is_continuation_if_runningEEEEESI_mEESK_EEE11do_completeEPvPNS0_19scheduler_operationERKSI_m+0xC2) [0x9a40f2]
      [js_test:auto2] 2016-03-05T06:16:48.708+0000 s20265|  mongos(_ZN4asio6detail14strand_service8dispatchINS0_17rewrapped_handlerINS0_7binder2INS0_7read_opINS_19basic_stream_socketINS_2ip3tcpENS_21stream_socket_serviceIS8_EEEENS_17mutable_buffers_1ENS0_14transfer_all_tENS0_15wrapped_handlerINS_10io_service6strandENSt3__18functionIFvNSH_10error_codeEmEEENS0_26is_continuation_if_runningEEEEESJ_mEESL_EEEEvRPNS1_11strand_implERT_+0x13D) [0x9a3fad]
      [js_test:auto2] 2016-03-05T06:16:48.708+0000 s20265|  mongos(_ZN4asio6detail19asio_handler_invokeINS0_7binder2INS0_7read_opINS_19basic_stream_socketINS_2ip3tcpENS_21stream_socket_serviceIS6_EEEENS_17mutable_buffers_1ENS0_14transfer_all_tENS0_15wrapped_handlerINS_10io_service6strandENSt3__18functionIFvNSF_10error_codeEmEEENS0_26is_continuation_if_runningEEEEESH_mEESE_SJ_SK_EEvRT_PNSC_IT0_T1_T2_EE+0xF3) [0x9a3dc3]
      [js_test:auto2] 2016-03-05T06:16:48.709+0000 s20265|  mongos(_ZN4asio6detail23reactive_socket_recv_opINS_17mutable_buffers_1ENS0_7read_opINS_19basic_stream_socketINS_2ip3tcpENS_21stream_socket_serviceIS6_EEEES2_NS0_14transfer_all_tENS0_15wrapped_handlerINS_10io_service6strandENSt3__18functionIFvNSE_10error_codeEmEEENS0_26is_continuation_if_runningEEEEEE11do_completeEPvPNS0_19scheduler_operationERKSG_m+0x109) [0x9a3c39]
      [js_test:auto2] 2016-03-05T06:16:48.709+0000 s20265|  mongos(_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKNSt3__110error_codeE+0x282) [0xbec3a2]
      [js_test:auto2] 2016-03-05T06:16:48.709+0000 s20265|  mongos(_ZN4asio6detail9scheduler3runERNSt3__110error_codeE+0xC4) [0xbe3cb4]
      [js_test:auto2] 2016-03-05T06:16:48.709+0000 s20265|  mongos(_ZN4asio10io_service3runEv+0x2B) [0xbe3b6b]
      [js_test:auto2] 2016-03-05T06:16:48.709+0000 s20265|  mongos(+0x5B8DA4) [0x9b8da4]
      [js_test:auto2] 2016-03-05T06:16:48.709+0000 s20265|  libpthread.so.0(+0x80A5) [0x7f741d32b0a5]
      [js_test:auto2] 2016-03-05T06:16:48.709+0000 s20265|  libc.so.6(clone+0x6D) [0x7f741d058cfd]
      [js_test:auto2] 2016-03-05T06:16:48.709+0000 s20265| -----  END BACKTRACE  -----
      
      [js_test:auto2] 2016-03-05T06:16:49.159+0000 2016-03-05T06:16:49.158+0000 I -        [thread1] shell: stopped mongo program on port 20265
      [js_test:auto2] 2016-03-05T06:16:49.159+0000 2016-03-05T06:16:49.158+0000 E QUERY    [thread1] StopError: MongoDB process on port 20265 exited with error code -6 :
      [js_test:auto2] 2016-03-05T06:16:49.159+0000 StopError: MongoDB process on port 20265 exited with error code -6
      [js_test:auto2] 2016-03-05T06:16:49.159+0000 MongoRunner.StopError@src/mongo/shell/servers.js:694:44
      [js_test:auto2] 2016-03-05T06:16:49.159+0000 MongoRunner.stopMongod@src/mongo/shell/servers.js:774:1
      [js_test:auto2] 2016-03-05T06:16:49.159+0000 ShardingTest/this.stopMongos@src/mongo/shell/shardingtest.js:685:13
      [js_test:auto2] 2016-03-05T06:16:49.159+0000 ShardingTest/this.stop@src/mongo/shell/shardingtest.js:295:13
      [js_test:auto2] 2016-03-05T06:16:49.160+0000 @jstests/sharding/auto2.js:149:1
      [js_test:auto2] 2016-03-05T06:16:49.160+0000 @jstests/sharding/auto2.js:1:2
      [js_test:auto2] 2016-03-05T06:16:49.160+0000 
      [js_test:auto2] 2016-03-05T06:16:49.160+0000 failed to load: jstests/sharding/auto2.js
      

      Note that Adam's changes for SERVER-22950 may stress the shutdown code more frequently.

      BF Ticket Generated by charlie.swanson

            Assignee:
            schwerin@mongodb.com Andy Schwerin
            Reporter:
            schwerin@mongodb.com Andy Schwerin
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

              Created:
              Updated:
              Resolved: