Details
Description
2017-03-06T20:02:22.505+0800 I NETWORK [conn4063980] end connection 192.168.xx.33:44020 (102 connections now open)
|
2017-03-06T20:02:22.891+0800 E STORAGE [repl writer worker 1] WiredTiger (0) [1488801742:891740][28613:0x7f37b5105700], file:apollo_modb/index-84-9204097186848499981.wt, WT_CURSOR.remove: read checksum error for 4096B block at offset 95326208: calculated block checksum of 1332835297 doesn't match expected checksum of 3877724731
|
2017-03-06T20:02:22.891+0800 E STORAGE [repl writer worker 1] WiredTiger (0) [1488801742:891802][28613:0x7f37b5105700], file:apollo_modb/index-84-9204097186848499981.wt, WT_CURSOR.remove: apollo_modb/index-84-9204097186848499981.wt: encountered an illegal file format or internal value
|
2017-03-06T20:02:22.891+0800 E STORAGE [repl writer worker 1] WiredTiger (-31804) [1488801742:891813][28613:0x7f37b5105700], file:apollo_modb/index-84-9204097186848499981.wt, WT_CURSOR.remove: the process must exit and restart: WT_PANIC: WiredTiger library panic
|
2017-03-06T20:02:22.891+0800 I - [repl writer worker 1] Fatal Assertion 28558
|
2017-03-06T20:02:22.891+0800 I - [repl writer worker 1]
|
|
|
***aborting after fassert() failure
|
|
|
|
|
2017-03-06T20:02:22.891+0800 I - [rsSync] Fatal Assertion 28559
|
2017-03-06T20:02:22.891+0800 I - [rsSync]
|
|
|
***aborting after fassert() failure
|
|
|
|
|
2017-03-06T20:02:22.970+0800 I - [WTJournalFlusher] Fatal Assertion 28559
|
2017-03-06T20:02:22.970+0800 I - [WTJournalFlusher]
|
|
|
***aborting after fassert() failure
|
|
|
|
|
2017-03-06T20:02:23.000+0800 I - [ftdc] Fatal Assertion 28559
|
2017-03-06T20:02:23.000+0800 I - [ftdc]
|
|
|
***aborting after fassert() failure
|
|
|
|
|
2017-03-06T20:02:23.027+0800 F - [repl writer worker 1] Got signal: 6 (Aborted).
|
|
|
0x13403e2 0x133f319 0x133fb22 0x7f37cff27710 0x7f37cfbb6625 0x7f37cfbb7e05 0x12c3602 0x10b3c63 0x1aad81c 0x1aadc9d 0x1aae084 0x19ddff2 0x19fb50d 0x1a00ab2 0x1a20b27 0x19f2244 0x1a3cc0a 0x1090a85 0x108f2f7 0xcade28 0xad7384 0xc23611 0xc23ff8 0xe32a05 0xe330c9 0xe331c5 0xd3d569 0xec4374 0xf5ac0e 0xf54a10 0xf56656 0xf57256 0xf5a55b 0x12ce131 0x12cea99 0x12cf5f0 0x1b69250 0x7f37cff1f9d1 0x7f37cfc6c8fd
|
----- BEGIN BACKTRACE -----
|
{"backtrace":[{"b":"400000","o":"F403E2","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"F3F319"},{"b":"400000","o":"F3FB22"},{"b":"7F37CFF18000","o":"F710"},{"b":"7F37CFB84000","o":"32625","s":"gsignal"},{"b":"7F37CFB84000","o":"33E05","s":"abort"},{"b":"400000","o":"EC3602","s":"_ZN5mongo13fassertFailedEi"},{"b":"400000","o":"CB3C63"},{"b":"400000","o":"16AD81C","s":"__wt_eventv"},{"b":"400000","o":"16ADC9D","s":"__wt_err"},{"b":"400000","o":"16AE084","s":"__wt_panic"},{"b":"400000","o":"15DDFF2","s":"__wt_bm_read"},{"b":"400000","o":"15FB50D","s":"__wt_bt_read"},{"b":"400000","o":"1600AB2","s":"__wt_page_in_func"},{"b":"400000","o":"1620B27","s":"__wt_row_search"},{"b":"400000","o":"15F2244","s":"__wt_btcur_remove"},{"b":"400000","o":"163CC0A"},{"b":"400000","o":"C90A85","s":"_ZN5mongo23WiredTigerIndexStandard8_unindexEP11__wt_cursorRKNS_7BSONObjERKNS_8RecordIdEb"},{"b":"400000","o":"C8F2F7","s":"_ZN5mongo15WiredTigerIndex7unindexEPNS_16OperationContextERKNS_7BSONObjERKNS_8RecordIdEb"},{"b":"400000","o":"8ADE28","s":"_ZN5mongo17IndexAccessMethod6updateEPNS_16OperationContextERKNS_12UpdateTicketEPl"},{"b":"400000","o":"6D7384","s":"_ZN5mongo10Collection14updateDocumentEPNS_16OperationContextERKNS_8RecordIdERKNS_11SnapshottedINS_7BSONObjEEERKS7_bbPNS_7OpDebugERNS_20oplogUpdateEntryArgsE"},{"b":"400000","o":"823611","s":"_ZN5mongo11UpdateStage18transformAndUpdateERKNS_11SnapshottedINS_7BSONObjEEERNS_8RecordIdE"},{"b":"400000","o":"823FF8","s":"_ZN5mongo11UpdateStage4workEPm"},{"b":"400000","o":"A32A05","s":"_ZN5mongo12PlanExecutor11getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE"},{"b":"400000","o":"A330C9","s":"_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE"},{"b":"400000","o":"A331C5","s":"_ZN5mongo12PlanExecutor11executePlanEv"},{"b":"400000","o":"93D569","s":"_ZN5mongo6updateEPNS_16OperationContextEPNS_8DatabaseERKNS_13UpdateRequestEPNS_7OpDebugE"},{"b":"400000","o":"AC4374","s":"_ZN5mongo4repl21applyOperation_inlockEPNS_16OperationContextEPNS_8DatabaseERKNS_7BSONObjEb"},{"b":"400000","o":"B5AC0E","s":"_ZNSt17_Function_handlerIFN5mongo6StatusEPNS0_16OperationContextEPNS0_8DatabaseERKNS0_7BSONObjEbEPS9_E9_M_invokeERKSt9_Any_dataS3_S5_S8_b"},{"b":"400000","o":"B54A10"},{"b":"400000","o":"B56656","s":"_ZN5mongo4repl8SyncTail9syncApplyEPNS_16OperationContextERKNS_7BSONObjEbSt8functionIFNS_6StatusES3_PNS_8DatabaseES6_bEES7_IFS8_S3_S6_EES7_IFvvEE"},{"b":"400000","o":"B57256","s":"_ZN5mongo4repl8SyncTail9syncApplyEPNS_16OperationContextERKNS_7BSONObjEb"},{"b":"400000","o":"B5A55B","s":"_ZN5mongo4repl14multiSyncApplyERKSt6vectorINS_7BSONObjESaIS2_EEPNS0_8SyncTailE"},{"b":"400000","o":"ECE131","s":"_ZN5mongo10ThreadPool10_doOneTaskEPSt11unique_lockISt5mutexE"},{"b":"400000","o":"ECEA99","s":"_ZN5mongo10ThreadPool13_consumeTasksEv"},{"b":"400000","o":"ECF5F0","s":"_ZN5mongo10ThreadPool17_workerThreadBodyEPS0_RKSs"},{"b":"400000","o":"1769250","s":"execute_native_thread_routine"},{"b":"7F37CFF18000","o":"79D1"},{"b":"7F37CFB84000","o":"E88FD","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.2.9", "gitVersion" : "22ec9e93b40c85fc7cae7d56e7d6a02fd811088c", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "2.6.32-504.16.2.el6.x86_64", "version" : "#1 SMP Wed Apr 22 06:48:29 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "C3B1D60B7051B5D77B97AA9E4D263F39E90B2632" }, { "b" : "7FFF1D2FF000", "elfType" : 3, "buildId" : "044DC84B6E7F9EDDF8698512A1754429564EC692" }, { "b" : "7F37D0DBF000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "7CE5058BE3F2AE6EBC0DF61E790C3FCED1A34C7B" }, { "b" : "7F37D09DB000", "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "096F60DC43484ECA364206125CED7D563FF9D027" }, { "b" : "7F37D07D3000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "583411D8786F86A1D6B8741C502831E6122445A7" }, { "b" : "7F37D05CF000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "454F8FC6CC6502C6401E5F9E221564D80665D277" }, { "b" : "7F37D034B000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "C9A87F6A29ED1D3CB18F539845A45FE3A9877FF1" }, { "b" : "7F37D0135000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "9A6E4BDFA184364D81F7DFD789474C3FB8F98A00" }, { "b" : "7F37CFF18000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "B8DFF8E53D9F2B80C3C382E83EC17C828B536A39" }, { "b" : "7F37CFB84000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "1425CB3B4C2F49C8101ED9B8F1D289053B4DFA77" }, { "b" : "7F37D102C000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "6F8E59B70E469F3A924A268911FF8FD0C37E7460" }, { "b" : "7F37CF940000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "9B852585C66329AA02EFB28497E652A40F538E78" }, { "b" : "7F37CF659000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "EF3AACAFD6BF71BB861F194C1559153FB0B020E2" }, { "b" : "7F37CF455000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "152E2C18A7A2145021A8A879A01A82EE134E3946" }, { "b" : "7F37CF229000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "DDE6774979156442185836150FC0785170F8001F" }, { "b" : "7F37CF013000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "b" : "7F37CEE08000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A23DAFBCE170763BF1E836A8B26113F9CD20E0DA" }, { "b" : "7F37CEC05000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "b" : "7F37CE9EB000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "58B696478044E028A5970D48A4ED50E164B43B36" }, { "b" : "7F37CE7CC000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "B4576BE308DDCF7BC31F7304E4734C3D846D0236" } ] }}
|
mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x13403e2]
|
mongod(+0xF3F319) [0x133f319]
|
mongod(+0xF3FB22) [0x133fb22]
|
libpthread.so.0(+0xF710) [0x7f37cff27710]
|
libc.so.6(gsignal+0x35) [0x7f37cfbb6625]
|
libc.so.6(abort+0x175) [0x7f37cfbb7e05]
|
mongod(_ZN5mongo13fassertFailedEi+0x82) [0x12c3602]
|
mongod(+0xCB3C63) [0x10b3c63]
|
mongod(__wt_eventv+0x42C) [0x1aad81c]
|
mongod(__wt_err+0x8D) [0x1aadc9d]
|
mongod(__wt_panic+0x24) [0x1aae084]
|
mongod(__wt_bm_read+0x82) [0x19ddff2]
|
mongod(__wt_bt_read+0x1ED) [0x19fb50d]
|
mongod(__wt_page_in_func+0x202) [0x1a00ab2]
|
mongod(__wt_row_search+0x687) [0x1a20b27]
|
mongod(__wt_btcur_remove+0x304) [0x19f2244]
|
mongod(+0x163CC0A) [0x1a3cc0a]
|
mongod(_ZN5mongo23WiredTigerIndexStandard8_unindexEP11__wt_cursorRKNS_7BSONObjERKNS_8RecordIdEb+0xC5) [0x1090a85]
|
mongod(_ZN5mongo15WiredTigerIndex7unindexEPNS_16OperationContextERKNS_7BSONObjERKNS_8RecordIdEb+0x77) [0x108f2f7]
|
mongod(_ZN5mongo17IndexAccessMethod6updateEPNS_16OperationContextERKNS_12UpdateTicketEPl+0xB8) [0xcade28]
|
mongod(_ZN5mongo10Collection14updateDocumentEPNS_16OperationContextERKNS_8RecordIdERKNS_11SnapshottedINS_7BSONObjEEERKS7_bbPNS_7OpDebugERNS_20oplogUpdateEntryArgsE+0x984) [0xad7384]
|
mongod(_ZN5mongo11UpdateStage18transformAndUpdateERKNS_11SnapshottedINS_7BSONObjEEERNS_8RecordIdE+0xAD1) [0xc23611]
|
mongod(_ZN5mongo11UpdateStage4workEPm+0x5D8) [0xc23ff8]
|
mongod(_ZN5mongo12PlanExecutor11getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0x275) [0xe32a05]
|
mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x39) [0xe330c9]
|
mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x55) [0xe331c5]
|
mongod(_ZN5mongo6updateEPNS_16OperationContextEPNS_8DatabaseERKNS_13UpdateRequestEPNS_7OpDebugE+0x179) [0xd3d569]
|
mongod(_ZN5mongo4repl21applyOperation_inlockEPNS_16OperationContextEPNS_8DatabaseERKNS_7BSONObjEb+0x924) [0xec4374]
|
mongod(_ZNSt17_Function_handlerIFN5mongo6StatusEPNS0_16OperationContextEPNS0_8DatabaseERKNS0_7BSONObjEbEPS9_E9_M_invokeERKSt9_Any_dataS3_S5_S8_b+0x1E) [0xf5ac0e]
|
mongod(+0xB54A10) [0xf54a10]
|
mongod(_ZN5mongo4repl8SyncTail9syncApplyEPNS_16OperationContextERKNS_7BSONObjEbSt8functionIFNS_6StatusES3_PNS_8DatabaseES6_bEES7_IFS8_S3_S6_EES7_IFvvEE+0x336) [0xf56656]
|
mongod(_ZN5mongo4repl8SyncTail9syncApplyEPNS_16OperationContextERKNS_7BSONObjEb+0xE6) [0xf57256]
|
mongod(_ZN5mongo4repl14multiSyncApplyERKSt6vectorINS_7BSONObjESaIS2_EEPNS0_8SyncTailE+0x9B) [0xf5a55b]
|
mongod(_ZN5mongo10ThreadPool10_doOneTaskEPSt11unique_lockISt5mutexE+0x121) [0x12ce131]
|
mongod(_ZN5mongo10ThreadPool13_consumeTasksEv+0xA9) [0x12cea99]
|
mongod(_ZN5mongo10ThreadPool17_workerThreadBodyEPS0_RKSs+0x100) [0x12cf5f0]
|
mongod(execute_native_thread_routine+0x20) [0x1b69250]
|
libpthread.so.0(+0x79D1) [0x7f37cff1f9d1]
|
libc.so.6(clone+0x6D) [0x7f37cfc6c8fd]
|
----- END BACKTRACE -----
|