Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-38763

Mongodb restarts with the Got signal: 6 (Aborted)

    • Type: Icon: Question Question
    • Resolution: Done
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: 3.2.18
    • Component/s: Admin
    • Labels:
      None

      After the mongodb server suddenly loses power, it restarts with the Got signal: 6 (Aborted)

      Single Mode
      Got signal: 6 (Aborted) 

      Most recent stacktrace is here:


      2018-12-21T14:25:48.459+0800 I STORAGE [initandlisten] wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=5,log_size=2GB),statistics_log=(wait=0),verbose=(recovery_progress),
      2018-12-21T14:25:48.472+0800 E STORAGE [initandlisten] WiredTiger (0) [1545373548:472201][2895:0x7fa186f5ddc0], file:WiredTiger.wt, connection: WiredTiger.turtle: encountered an illegal file format or internal value
      2018-12-21T14:25:48.472+0800 E STORAGE [initandlisten] WiredTiger (-31804) [1545373548:472257][2895:0x7fa186f5ddc0], file:WiredTiger.wt, connection: the process must exit and restart: WT_PANIC: WiredTiger library panic
      2018-12-21T14:25:48.472+0800 I - [initandlisten] Fatal Assertion 28558
      2018-12-21T14:25:48.472+0800 I - [initandlisten]

      ***aborting after fassert() failure

      2018-12-21T14:25:48.491+0800 F - [initandlisten] Got signal: 6 (Aborted).

      0x1333ed2 0x1333029 0x1333832 0x7fa185844100 0x7fa1854a9597 0x7fa1854aac88 0x12b45c2 0x10a8bb3 0x95c71e 0x95c91b 0x95cadf 0x1a6427f 0x1a62c5a 0x1a1bfe4 0x1aa101d 0x1aa1539 0x1aa175b 0x1a298a6 0x1a9df2d 0x1a6242f 0x1a6252e 0x1a1887a 0x1090116 0x108c3e3 0xfb521e 0x9a7400 0x95d31d 0x7fa185495bb5 0x9a2e0d
      ----- BEGIN BACKTRACE -----

      {"backtrace":[\{"b":"400000","o":"F33ED2","s":"_ZN5mongo15printStackTraceERSo"}

      ,{"b":"400000","o":"F33029"},{"b":"400000","o":"F33832"},{"b":"7FA185835000","o":"F100"},{"b":"7FA185474000","o":"35597","s":"gsignal"},{"b":"7FA185474000","o":"36C88","s":"abort"},{"b":"400000","o":"EB45C2","s":"ZN5mongo13fassertFailedEi"},{"b":"400000","o":"CA8BB3"},{"b":"400000","o":"55C71E","s":"wt_eventv"},{"b":"400000","o":"55C91B","s":"wt_err"},{"b":"400000","o":"55CADF","s":"wt_panic"},{"b":"400000","o":"166427F","s":"wt_turtle_read"},{"b":"400000","o":"1662C5A","s":"wt_metadata_search"},{"b":"400000","o":"161BFE4","s":"wt_conn_btree_open"},{"b":"400000","o":"16A101D","s":"wt_session_get_btree"},{"b":"400000","o":"16A1539","s":"wt_session_get_btree"},{"b":"400000","o":"16A175B","s":"wt_session_get_btree_ckpt"},{"b":"400000","o":"16298A6","s":"wt_curfile_open"},{"b":"400000","o":"169DF2D"},{"b":"400000","o":"166242F","s":"wt_metadata_cursor_open"},{"b":"400000","o":"166252E","s":"wt_metadata_cursor"},{"b":"400000","o":"161887A","s":"wiredtiger_open"},{"b":"400000","o":"C90116","s":"_ZN5mongo18WiredTigerKVEngineC2ERKSsS2_S2_mbbb"},{"b":"400000","o":"C8C3E3"},{"b":"400000","o":"BB521E","s":"_ZN5mongo20ServiceContextMongoD29initializeGlobalStorageEngineEv"},{"b":"400000","o":"5A7400"},{"b":"400000","o":"55D31D","s":"main"},{"b":"7FA185474000","o":"21BB5","s":"_libc_start_main"},{"b":"400000","o":"5A2E0D"}],"processInfo":{ "mongodbVersion" : "3.2.18", "gitVersion" : "4c1bae566c0c00f996a2feb16febf84936ecaf6f", "compiledModules" : [], "uname" :

      { "sysname" : "Linux", "release" : "3.10.0-514.44.5.10.h148.x86_64", "version" : "#1 SMP Tue Oct 16 13:33:50 UTC 2018", "machine" : "x86_64" }

      , "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "B67877A4029ADCB7A1D0EFCD5EC4E741170CBB5C" }, { "b" : "7FFF038F7000", "elfType" : 3, "buildId" : "186FFDDE5A7B05EDC21076991E5BA22C7FF6BEA7" }, { "b" : "7FA186ADF000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "FC055EF145EFF43D9661FD531872196873E4C4DB" }, { "b" : "7FA18667E000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F8A23B6DC3E995C3AAED11CBEE40F0A32E30BAF5" }, { "b" : "7FA186476000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "9BDBE5FE8313D6AC869AAECB70428E41AF76685F" }, { "b" : "7FA186272000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "926DBA873EFA28B2B12B5F42A5D56CC2785E36A4" }, { "b" : "7FA185F69000", "path" : "/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "CF370C0D1A3D357D8A550FD0620722DC9551C157" }, { "b" : "7FA185C67000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "B5FAA069E09135A3F1486DD8931E01B8976E42A7" }, { "b" : "7FA185A51000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "3B7E79322B4B032A5AE40B0E7A6C4297059C9FD8" }, { "b" : "7FA185835000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "6DE18DE37DCBA5C51745D477FB07C64E2728606D" }, { "b" : "7FA185474000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "D66907E44C19B76013E111310D73473A9C265E0D" }, { "b" : "7FA186D51000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "5460CA1B85820630FE5171A049178C9774B1BB12" }, { "b" : "7FA185226000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "96B4B54E6C3F014847249652A25D5D082BE95654" }, { "b" : "7FA184F3F000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "C18C0FEDB5E38502A266C32C2A5057FBD389E773" }, { "b" : "7FA184D3B000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "34625B0B43E97268B8B12C9F796AE6C321D5447B" }, { "b" : "7FA184B09000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "C80591A09A605BD029B6FA9B30B4B0ABA1E07C30" }, { "b" : "7FA1848EE000", "path" : "/home/lib/libz.so.1", "elfType" : 3, "buildId" : "5EDC02185C38EF32E79414B531C6BC6932CB99C6" }, { "b" : "7FA1846DF000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "C6FE9AD02CE8511BD9AC03E7D6FDC3BE95995DBA" }, { "b" : "7FA1844DB000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "E7FD292753065CB083D21C743F998F9ECC824B1C" }, { "b" : "7FA1842C1000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "33CC61E1C3203B99DB184210060FADB4B582001D" }, { "b" : "7FA18409C000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "DA49E0D5C8E9150C3860617509A0CD6106E487A0" }, { "b" : "7FA183E3B000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "7D76D546405803D934E009061E6DECD4874EA63E" }, { "b" : "7FA183C16000", "path" : "/lib64/liblzma.so.5", "elfType" : 3, "buildId" : "0815E89D27D798AB2F45CA887DF9B3085E3BAB52" } ] }}
      mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x1333ed2]
      mongod(+0xF33029) [0x1333029]
      mongod(+0xF33832) [0x1333832]
      libpthread.so.0(+0xF100) [0x7fa185844100]
      libc.so.6(gsignal+0x37) [0x7fa1854a9597]
      libc.so.6(abort+0x148) [0x7fa1854aac88]
      mongod(_ZN5mongo13fassertFailedEi+0x82) [0x12b45c2]
      mongod(+0xCA8BB3) [0x10a8bb3]
      mongod(__wt_eventv+0x440) [0x95c71e]
      mongod(__wt_err+0x8D) [0x95c91b]
      mongod(__wt_panic+0x24) [0x95cadf]
      mongod(__wt_turtle_read+0x24F) [0x1a6427f]
      mongod(__wt_metadata_search+0x8A) [0x1a62c5a]
      mongod(__wt_conn_btree_open+0x64) [0x1a1bfe4]
      mongod(__wt_session_get_btree+0xDD) [0x1aa101d]
      mongod(__wt_session_get_btree+0x5F9) [0x1aa1539]
      mongod(__wt_session_get_btree_ckpt+0xAB) [0x1aa175b]
      mongod(__wt_curfile_open+0x86) [0x1a298a6]
      mongod(+0x169DF2D) [0x1a9df2d]
      mongod(__wt_metadata_cursor_open+0x5F) [0x1a6242f]
      mongod(__wt_metadata_cursor+0x7E) [0x1a6252e]
      mongod(wiredtiger_open+0x191A) [0x1a1887a]
      mongod(_ZN5mongo18WiredTigerKVEngineC2ERKSsS2_S2_mbbb+0x5B6) [0x1090116]
      mongod(+0xC8C3E3) [0x108c3e3]
      mongod(_ZN5mongo20ServiceContextMongoD29initializeGlobalStorageEngineEv+0x59E) [0xfb521e]
      mongod(+0x5A7400) [0x9a7400]
      mongod(main+0x15D) [0x95d31d]
      libc.so.6(__libc_start_main+0xF5) [0x7fa185495bb5]
      mongod(+0x5A2E0D) [0x9a2e0d]


       

      Attachments are WiredTiger.wt and WiredTiger.turtle

      Can you fix these two files for me?

        1. data.rar
          10 kB
        2. repair_attempt.tar.gz
          19 kB

            Assignee:
            kelsey.schubert@mongodb.com Kelsey Schubert
            Reporter:
            mr-liu Liu Pengshi
            Votes:
            0 Vote for this issue
            Watchers:
            5 Start watching this issue

              Created:
              Updated:
              Resolved: