Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-38633

Fatal Assertion 50833 at src/mongo/transport/baton_asio_linux.h 68

    XMLWordPrintable

    Details

    • Type: Bug
    • Status: Closed
    • Priority: Major - P3
    • Resolution: Works as Designed
    • Affects Version/s: 4.0.4
    • Fix Version/s: None
    • Component/s: Sharding
    • Labels:
      None
    • Operating System:
      ALL
    • Steps To Reproduce:
      Hide

      1 - RUN mongos --configdb bigcachecfg/10.132.0.16:27017 --bind_ip_all

      2 - insert many documents

      Show
      1 - RUN mongos --configdb bigcachecfg/10.132.0.16:27017 --bind_ip_all 2 - insert many documents
    • Sprint:
      Service Arch 2018-12-17, Service Arch 2018-12-31, Service Arch 2019-01-14

      Description

      Hello everybody,

       

      I am currently setup a shard mongo cluster in GCP compute engine (google cloud) with 3 shard, 1 mongo config, and 1 mongo router.

       

      Shard seems working well but i have issue with the mongo router (mongos).

      It crashes every 2 min during heavy load (125 000 docs/s)

      Ram and Cpu of the router are ok.

       

      I have the same issue with --nounixsocket parameter

       

      2018-12-14T09:50:33.327+0000 F NETWORK [conn14] error in eventfd: Unknown error 2018-12-14T09:50:33.327+0000 F - [conn14] Fatal Assertion 50833 at src/mongo/transport/baton_asio_linux.h 68
      2018-12-14T09:50:33.327+0000 F - [conn14] ***aborting after fassert() failure 2018-12-14T09:50:33.339+0000 F - [conn14] Got signal: 6 (Aborted). 0x55d43de28cd1 0x55d43de27ee9 0x55d43de283cd 0x7f74a625c0c0 0x7f74a5edefff 0x7f74a5ee042a 0x55d43d1f6221 0x55d43d6c62bb 0x55d43d6b8ac9 0x55d43d47266b 0x55d43d475a4b 0x55d43d3583fd 0x55d43d35650c 0x55d43d33074d 0x55d43d349bb5 0x55d43d34aa42 0x55d43d249327 0x55d43d2683aa 0x55d43d263037 0x55d43d266881 0x55d43d6c37f2 0x55d43d261220 0x55d43d264405 0x55d43d262777 0x55d43d2630bd 0x55d43d266881 0x55d43d6c3d55 0x55d43dda4244 0x7f74a6252494 0x7f74a5f94acf ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"55D43CCA8000","o":"1180CD1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"55D43CCA8000","o":"117FEE9"},{"b":"55D43CCA8000","o":"11803CD"},{"b":"7F74A624B000","o":"110C0"},{"b":"7F74A5EAC000","o":"32FFF","s":"gsignal"},{"b":"7F74A5EAC000","o":"3442A","s":"abort"},{"b":"55D43CCA8000","o":"54E221","s":"_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj"},{"b":"55D43CCA8000","o":"A1E2BB","s":"_ZN5mongo9transport18TransportLayerASIO9makeBatonEPNS_16OperationContextE"},{"b":"55D43CCA8000","o":"A10AC9","s":"_ZN5mongo9transport21TransportLayerManager9makeBatonEPNS_16OperationContextE"},{"b":"55D43CCA8000","o":"7CA66B","s":"_ZN5mongo19AsyncRequestsSender13BatonDetacherC2EPNS_16OperationContextE"},{"b":"55D43CCA8000","o":"7CDA4B","s":"_ZN5mongo19AsyncRequestsSenderC2EPNS_16OperationContextEPNS_8executor12TaskExecutorENS_10StringDataERKSt6vectorINS0_7RequestESaIS8_EERKNS_21ReadPreferenceSettingENS_5Shard11RetryPolicyE"},{"b":"55D43CCA8000","o":"6B03FD","s":"_ZN5mongo14BatchWriteExec12executeBatchEPNS_16OperationContextERNS_10NSTargeterERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseEPNS_19BatchWriteExecStatsE"},{"b":"55D43CCA8000","o":"6AE50C","s":"_ZN5mongo13ClusterWriter5writeEPNS_16OperationContextERKNS_21BatchedCommandRequestEPNS_19BatchWriteExecStatsEPNS_22BatchedCommandResponseE"},{"b":"55D43CCA8000","o":"68874D"},{"b":"55D43CCA8000","o":"6A1BB5"},{"b":"55D43CCA8000","o":"6A2A42","s":"_ZN5mongo8Strategy13clientCommandEPNS_16OperationContextERKNS_7MessageE"},{"b":"55D43CCA8000","o":"5A1327","s":"_ZN5mongo23ServiceEntryPointMongos13handleRequestEPNS_16OperationContextERKNS_7MessageE"},{"b":"55D43CCA8000","o":"5C03AA","s":"_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE"},{"b":"55D43CCA8000","o":"5BB037","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"55D43CCA8000","o":"5BE881"},{"b":"55D43CCA8000","o":"A1B7F2","s":"_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE"},{"b":"55D43CCA8000","o":"5B9220","s":"_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE"},{"b":"55D43CCA8000","o":"5BC405","s":"_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE"},{"b":"55D43CCA8000","o":"5BA777","s":"_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE"},{"b":"55D43CCA8000","o":"5BB0BD","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"55D43CCA8000","o":"5BE881"},{"b":"55D43CCA8000","o":"A1BD55"},{"b":"55D43CCA8000","o":"10FC244"},{"b":"7F74A624B000","o":"7494"},{"b":"7F74A5EAC000","o":"E8ACF","s":"clone"}],"processInfo":{ "mongodbVersion" : "4.0.4", "gitVersion" : "f288a3bdf201007f3693c58e140056adf8b04839", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "4.9.0-8-amd64", "version" : "#1 SMP Debian 4.9.130-2 (2018-10-27)", "machine" : "x86_64" }, "somap" : [ { "b" : "55D43CCA8000", "elfType" : 3, "buildId" : "749E8DEA519382D62431A36FEFA894BAEDF538D2" }, { "b" : "7FFDC68F0000", "path" : "linux-vdso.so.1", "elfType" : 3, "buildId" : "0B1047D7814362A9CE0444D05F6284EDA613920B" }, { "b" : "7F74A7494000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "713D47D5F599289C0A91ADE8F0122B2B4AA78B2E" }, { "b" : "7F74A6FFB000", "path" : "/usr/lib/x86_64-linux-gnu/libcrypto.so.1.1", "elfType" : 3, "buildId" : "A214BCD55713CB1E0B9AA61C07319C9A83A2268C" }, { "b" : "7F74A6D8F000", "path" : "/usr/lib/x86_64-linux-gnu/libssl.so.1.1", "elfType" : 3, "buildId" : "2BEF491D3EF8E727DF943799D1309AA357BA7D4C" }, { "b" : "7F74A6B8B000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "B895F0831F623C5F23603401D4069F9F94C24761" }, { "b" : "7F74A6983000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "5D83E0642E645026DBB11F89F7DF7106BD821495" }, { "b" : "7F74A667F000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "1B95E3A8B8788B07E4F59EE69B1877F9DEB42033" }, { "b" : "7F74A6468000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "51AD5FD294CD6C813BED40717347A53434B80B7A" }, { "b" : "7F74A624B000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "4285CD3158DDE596765C747AE210AB6CBD258B22" }, { "b" : "7F74A5EAC000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "AA889E26A70F98FA8D230D088F7CC5BF43573163" }, { "b" : "7F74A76AB000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "263F909DBE11A66F7C6233E3FF0521148D9F8370" } ] }} mongos(_ZN5mongo15printStackTraceERSo+0x41) [0x55d43de28cd1]
      mongos(+0x117FEE9) [0x55d43de27ee9]
       mongos(+0x11803CD) [0x55d43de283cd]
       libpthread.so.0(+0x110C0) [0x7f74a625c0c0]
       libc.so.6(gsignal+0xCF) [0x7f74a5edefff]
       libc.so.6(abort+0x16A) [0x7f74a5ee042a]
       mongos(_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj+0x0) [0x55d43d1f6221]
       mongos(_ZN5mongo9transport18TransportLayerASIO9makeBatonEPNS_16OperationContextE+0x31B) [0x55d43d6c62bb]
       mongos(_ZN5mongo9transport21TransportLayerManager9makeBatonEPNS_16OperationContextE+0x69) [0x55d43d6b8ac9]
       mongos(_ZN5mongo19AsyncRequestsSender13BatonDetacherC2EPNS_16OperationContextE+0x9B) [0x55d43d47266b]
       mongos(_ZN5mongo19AsyncRequestsSenderC2EPNS_16OperationContextEPNS_8executor12TaskExecutorENS_10StringDataERKSt6vectorINS0_7RequestESaIS8_EERKNS_21ReadPreferenceSettingENS_5Shard11RetryPolicyE+0x5B) [0x55d43d475a4b]
       mongos(_ZN5mongo14BatchWriteExec12executeBatchEPNS_16OperationContextERNS_10NSTargeterERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseEPNS_19BatchWriteExecStatsE+0x10FD) [0x55d43d3583fd]
       mongos(_ZN5mongo13ClusterWriter5writeEPNS_16OperationContextERKNS_21BatchedCommandRequestEPNS_19BatchWriteExecStatsEPNS_22BatchedCommandResponseE+0xEEC) [0x55d43d35650c]
       mongos(+0x68874D) [0x55d43d33074d]
       mongos(+0x6A1BB5) [0x55d43d349bb5]
       mongos(_ZN5mongo8Strategy13clientCommandEPNS_16OperationContextERKNS_7MessageE+0x1D2) [0x55d43d34aa42]
       mongos(_ZN5mongo23ServiceEntryPointMongos13handleRequestEPNS_16OperationContextERKNS_7MessageE+0x627) [0x55d43d249327]
       mongos(_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE+0xBA) [0x55d43d2683aa]
       mongos(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x97) [0x55d43d263037]
       mongos(+0x5BE881) [0x55d43d266881]
       mongos(_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE+0x1A2) [0x55d43d6c37f2]
       mongos(_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE+0x150) [0x55d43d261220]
       mongos(_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE+0xB05) [0x55d43d264405]
       mongos(_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE+0x357) [0x55d43d262777]
       mongos(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x11D) [0x55d43d2630bd]
       mongos(+0x5BE881) [0x55d43d266881]
       mongos(+0xA1BD55) [0x55d43d6c3d55]
       mongos(+0x10FC244) [0x55d43dda4244]
       libpthread.so.0(+0x7494) [0x7f74a6252494]
       libc.so.6(clone+0x3F) [0x7f74a5f94acf]
      ----- END BACKTRACE -----
      

       

        Attachments

          Issue Links

            Activity

              People

              Assignee:
              jason.carey Jason Carey
              Reporter:
              mattal ATTAL Mathieu
              Participants:
              Votes:
              0 Vote for this issue
              Watchers:
              9 Start watching this issue

                Dates

                Created:
                Updated:
                Resolved: