[SERVER-38763] Mongodb restarts with the Got signal: 6 (Aborted) Created: 22/Dec/18 Updated: 27/Dec/18 Resolved: 24/Dec/18 |
|
| Status: | Closed |
| Project: | Core Server |
| Component/s: | Admin |
| Affects Version/s: | 3.2.18 |
| Fix Version/s: | None |
| Type: | Question | Priority: | Major - P3 |
| Reporter: | Liu Pengshi | Assignee: | Kelsey Schubert |
| Resolution: | Done | Votes: | 0 |
| Labels: | None | ||
| Remaining Estimate: | Not Specified | ||
| Time Spent: | Not Specified | ||
| Original Estimate: | Not Specified | ||
| Attachments: |
|
||||||||
| Issue Links: |
|
||||||||
| Participants: | |||||||||
| Description |
|
After the mongodb server suddenly loses power, it restarts with the Got signal: 6 (Aborted) Single Mode Most recent stacktrace is here: 2018-12-21T14:25:48.459+0800 I STORAGE [initandlisten] wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=5,log_size=2GB),statistics_log=(wait=0),verbose=(recovery_progress), ***aborting after fassert() failure 2018-12-21T14:25:48.491+0800 F - [initandlisten] Got signal: 6 (Aborted). 0x1333ed2 0x1333029 0x1333832 0x7fa185844100 0x7fa1854a9597 0x7fa1854aac88 0x12b45c2 0x10a8bb3 0x95c71e 0x95c91b 0x95cadf 0x1a6427f 0x1a62c5a 0x1a1bfe4 0x1aa101d 0x1aa1539 0x1aa175b 0x1a298a6 0x1a9df2d 0x1a6242f 0x1a6252e 0x1a1887a 0x1090116 0x108c3e3 0xfb521e 0x9a7400 0x95d31d 0x7fa185495bb5 0x9a2e0d ,{"b":"400000","o":"F33029"},{"b":"400000","o":"F33832"},{"b":"7FA185835000","o":"F100"},{"b":"7FA185474000","o":"35597","s":"gsignal"},{"b":"7FA185474000","o":"36C88","s":"abort"},{"b":"400000","o":"EB45C2","s":"ZN5mongo13fassertFailedEi"},{"b":"400000","o":"CA8BB3"},{"b":"400000","o":"55C71E","s":"wt_eventv"},{"b":"400000","o":"55C91B","s":"wt_err"},{"b":"400000","o":"55CADF","s":"wt_panic"},{"b":"400000","o":"166427F","s":"wt_turtle_read"},{"b":"400000","o":"1662C5A","s":"wt_metadata_search"},{"b":"400000","o":"161BFE4","s":"wt_conn_btree_open"},{"b":"400000","o":"16A101D","s":"wt_session_get_btree"},{"b":"400000","o":"16A1539","s":"wt_session_get_btree"},{"b":"400000","o":"16A175B","s":"wt_session_get_btree_ckpt"},{"b":"400000","o":"16298A6","s":"wt_curfile_open"},{"b":"400000","o":"169DF2D"},{"b":"400000","o":"166242F","s":"wt_metadata_cursor_open"},{"b":"400000","o":"166252E","s":"wt_metadata_cursor"},{"b":"400000","o":"161887A","s":"wiredtiger_open"},{"b":"400000","o":"C90116","s":"_ZN5mongo18WiredTigerKVEngineC2ERKSsS2_S2_mbbb"},{"b":"400000","o":"C8C3E3"},{"b":"400000","o":"BB521E","s":"_ZN5mongo20ServiceContextMongoD29initializeGlobalStorageEngineEv"},{"b":"400000","o":"5A7400"},{"b":"400000","o":"55D31D","s":"main"},{"b":"7FA185474000","o":"21BB5","s":"_libc_start_main"},{"b":"400000","o":"5A2E0D"}],"processInfo":{ "mongodbVersion" : "3.2.18", "gitVersion" : "4c1bae566c0c00f996a2feb16febf84936ecaf6f", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.10.0-514.44.5.10.h148.x86_64", "version" : "#1 SMP Tue Oct 16 13:33:50 UTC 2018", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "B67877A4029ADCB7A1D0EFCD5EC4E741170CBB5C" }, { "b" : "7FFF038F7000", "elfType" : 3, "buildId" : "186FFDDE5A7B05EDC21076991E5BA22C7FF6BEA7" }, { "b" : "7FA186ADF000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "FC055EF145EFF43D9661FD531872196873E4C4DB" }, { "b" : "7FA18667E000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F8A23B6DC3E995C3AAED11CBEE40F0A32E30BAF5" }, { "b" : "7FA186476000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "9BDBE5FE8313D6AC869AAECB70428E41AF76685F" }, { "b" : "7FA186272000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "926DBA873EFA28B2B12B5F42A5D56CC2785E36A4" }, { "b" : "7FA185F69000", "path" : "/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "CF370C0D1A3D357D8A550FD0620722DC9551C157" }, { "b" : "7FA185C67000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "B5FAA069E09135A3F1486DD8931E01B8976E42A7" }, { "b" : "7FA185A51000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "3B7E79322B4B032A5AE40B0E7A6C4297059C9FD8" }, { "b" : "7FA185835000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "6DE18DE37DCBA5C51745D477FB07C64E2728606D" }, { "b" : "7FA185474000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "D66907E44C19B76013E111310D73473A9C265E0D" }, { "b" : "7FA186D51000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "5460CA1B85820630FE5171A049178C9774B1BB12" }, { "b" : "7FA185226000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "96B4B54E6C3F014847249652A25D5D082BE95654" }, { "b" : "7FA184F3F000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "C18C0FEDB5E38502A266C32C2A5057FBD389E773" }, { "b" : "7FA184D3B000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "34625B0B43E97268B8B12C9F796AE6C321D5447B" }, { "b" : "7FA184B09000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "C80591A09A605BD029B6FA9B30B4B0ABA1E07C30" }, { "b" : "7FA1848EE000", "path" : "/home/lib/libz.so.1", "elfType" : 3, "buildId" : "5EDC02185C38EF32E79414B531C6BC6932CB99C6" }, { "b" : "7FA1846DF000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "C6FE9AD02CE8511BD9AC03E7D6FDC3BE95995DBA" }, { "b" : "7FA1844DB000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "E7FD292753065CB083D21C743F998F9ECC824B1C" }, { "b" : "7FA1842C1000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "33CC61E1C3203B99DB184210060FADB4B582001D" }, { "b" : "7FA18409C000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "DA49E0D5C8E9150C3860617509A0CD6106E487A0" }, { "b" : "7FA183E3B000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "7D76D546405803D934E009061E6DECD4874EA63E" }, { "b" : "7FA183C16000", "path" : "/lib64/liblzma.so.5", "elfType" : 3, "buildId" : "0815E89D27D798AB2F45CA887DF9B3085E3BAB52" } ] }}
Attachments are WiredTiger.wt and WiredTiger.turtle Can you fix these two files for me? |
| Comments |
| Comment by Liu Pengshi [ 26/Dec/18 ] |
|
Thank you for fixing the data, but there are still the same errors as last time. What else can you do for me to save this data? The following is the error message 2018-12-26T11:17:29.379+0800 I STORAGE [initandlisten] wiredtiger_open config: create,cache_size=4G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=5,log_size=2GB),statistics_log=(wait=0), ***aborting after fassert() failure 2018-12-26T11:17:29.412+0800 F - [initandlisten] Got signal: 6 (Aborted). 0x130ab72 0x1309cc9 0x130a4d2 0x7f04a9e3b100 0x7f04a9aa0597 0x7f04a9aa1c88 0x1291192 0x108b693 0x94dc9e 0x94dd73 0x94df37 0x1a3d606 0x1a3bf9a 0x19f30e1 0x1a79dfe 0x1a7a485 0x1a7a8fb 0x1a01d06 0x1a7001d 0x1a3b78f 0x1a3b89e 0x19ef62e 0x1072ba7 0x106ee93 0xf98788 0x99613e 0x9986f0 0x94e79d 0x7f04a9a8cbb5 0x992aad ,{"b":"400000","o":"F09CC9"},{"b":"400000","o":"F0A4D2"},{"b":"7F04A9E2C000","o":"F100"},{"b":"7F04A9A6B000","o":"35597","s":"gsignal"},{"b":"7F04A9A6B000","o":"36C88","s":"abort"},{"b":"400000","o":"E91192","s":"ZN5mongo13fassertFailedEi"},{"b":"400000","o":"C8B693"},{"b":"400000","o":"54DC9E"},{"b":"400000","o":"54DD73","s":"wt_err"},{"b":"400000","o":"54DF37","s":"wt_panic"},{"b":"400000","o":"163D606"},{"b":"400000","o":"163BF9A"},{"b":"400000","o":"15F30E1"},{"b":"400000","o":"1679DFE"},{"b":"400000","o":"167A485"},{"b":"400000","o":"167A8FB"},{"b":"400000","o":"1601D06"},{"b":"400000","o":"167001D"},{"b":"400000","o":"163B78F"},{"b":"400000","o":"163B89E"},{"b":"400000","o":"15EF62E","s":"wiredtiger_open"},{"b":"400000","o":"C72BA7","s":"_ZN5mongo18WiredTigerKVEngineC2ERKSsS2_S2_mbbb"},{"b":"400000","o":"C6EE93"},{"b":"400000","o":"B98788","s":"_ZN5mongo20ServiceContextMongoD29initializeGlobalStorageEngineEv"},{"b":"400000","o":"59613E"},{"b":"400000","o":"5986F0","s":"_ZN5mongo13initAndListenEi"},{"b":"400000","o":"54E79D","s":"main"},{"b":"7F04A9A6B000","o":"21BB5","s":"_libc_start_main"},{"b":"400000","o":"592AAD"}],"processInfo":{ "mongodbVersion" : "3.2.12", "gitVersion" : "ef3e1bc78e997f0d9f22f45aeb1d8e3b6ac14a14", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.10.0-327.59.59.46.h24.x86_64", "version" : "#1 SMP Wed Oct 11 06:59:50 UTC 2017", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "ADE07DD69BB7FD3B7000A0A0FE3C8178E7C83809" }, { "b" : "7FFE71891000", "elfType" : 3, "buildId" : "385332720BD3BAAB7719A3D57F3FA52B9E70D20E" }, { "b" : "7F04AB0D8000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "AAB14F6D12976911FDFDA468E4F6213595F05798" }, { "b" : "7F04AACEE000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "8B2012EB69FF14394DDC8321F468022AD82B8482" }, { "b" : "7F04AAAE6000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "2D227847AB521134592AEC7BA0F89025137449ED" }, { "b" : "7F04AA8E2000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "0D169E3DB55D4C66617903F940E412803298DA80" }, { "b" : "7F04AA560000", "path" : "/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "3BCAF1CF6BBCC1C79A0A64B3E996BD365F11C71A" }, { "b" : "7F04AA25E000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "0AFFB8028C0403515173F63B973AFA45D8A6E7DD" }, { "b" : "7F04AA048000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "802D73D3C6B7203CF2CACBA3AF94A463E536FA57" }, { "b" : "7F04A9E2C000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "2DB99C0B19EB7547970E44D450444332D06EBAA8" }, { "b" : "7F04A9A6B000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "28D908BB0F165C9EA3B2501734B6E6CA619BDA7F" }, { "b" : "7F04AB346000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "5460CA1B85820630FE5171A049178C9774B1BB12" }, { "b" : "7F04A981D000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "A875D496A04DEECBE1B99645B2AB3AF9D57EAFDC" }, { "b" : "7F04A9536000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "83AC4B0EA72D0C262414C60A23841CA30FD4FAEB" }, { "b" : "7F04A9332000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "F91C59010F3EADFB0F37BD4A3D9F9DA7F2A5449E" }, { "b" : "7F04A9100000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "5A49BC797F1927AB1091AB74D2FCD5394DFEC678" }, { "b" : "7F04A8EEA000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "AAC85EE763F961681951A9D68D9F8A608178FEF2" }, { "b" : "7F04A8CDB000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "55AAF7A0C4220AD1E5D4A73D2AC33BB78279539F" }, { "b" : "7F04A8AD7000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "E7FD292753065CB083D21C743F998F9ECC824B1C" }, { "b" : "7F04A88BD000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "55A18352E2DB168A3EE70CFB1C553C138B8F7F5A" }, { "b" : "7F04A8698000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "DA49E0D5C8E9150C3860617509A0CD6106E487A0" }, { "b" : "7F04A8437000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "7D76D546405803D934E009061E6DECD4874EA63E" }, { "b" : "7F04A8212000", "path" : "/lib64/liblzma.so.5", "elfType" : 3, "buildId" : "0815E89D27D798AB2F45CA887DF9B3085E3BAB52" } ] }} Thank you, Liu |
| Comment by Kelsey Schubert [ 24/Dec/18 ] |
|
Hi mr-liu, Thank you for your report. I've attached a repair attempt, repair_attempt.tar.gz Thank you, |