-
Type:
Bug
-
Resolution: Unresolved
-
Priority:
Major - P3
-
None
-
Affects Version/s: None
-
Component/s: Reconciliation
-
Storage Engines
-
StorEng - 2024-11-12, StorEng - 2025-01-21
This issue occurred in Atlas on version 7.0.12 and has only happened once. Refer to the AF ticket linked for further information.
Scope: Investigate why the assertion was triggered and determine if there is a bug. If it is a bug, fix it as part of this ticket.
Assertion -
WT_ASSERT_ALWAYS(session, ts == 1 || ts == WT_TS_NONE, "out-of-order timestamp update detected");
Below is the relevant stack trace -
153830:{"t":{"$date":"2024-10-12T16:27:24.638+00:00"},"s":"E", "c":"WT", "id":22435, "ctx":"Checkpointer","msg":"WiredTiger error message","attr":{"error":0,"message":{"ts_sec":1728750444,"ts_usec":638307,"thread":"6620:0x7f132cbeb700","session_dhandle_name":"file:collection-791--6845604881235222348.wt","session_name":"WT_SESSION.checkpoint","category":"WT_VERB_DEFAULT","category_id":9,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"__hs_delete_reinsert_from_pos:922:WiredTiger assertion failed: 'ts == 1 || ts == 0'. out-of-order timestamp update detected"}}}
153831:{"t":{"$date":"2024-10-12T16:27:24.638+00:00"},"s":"E", "c":"WT", "id":22435, "ctx":"Checkpointer","msg":"WiredTiger error message","attr":{"error":0,"message":{"ts_sec":1728750444,"ts_usec":638369,"thread":"6620:0x7f132cbeb700","session_dhandle_name":"file:collection-791--6845604881235222348.wt","session_name":"WT_SESSION.checkpoint","category":"WT_VERB_DEFAULT","category_id":9,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"__wt_abort:28:aborting WiredTiger library"}}}
153832:{"t":{"$date":"2024-10-12T16:27:24.638+00:00"},"s":"F", "c":"CONTROL", "id":6384300, "ctx":"Checkpointer","msg":"Writing fatal message","attr":{"message":"\n"}}
153833:{"t":{"$date":"2024-10-12T16:27:24.638+00:00"},"s":"F", "c":"CONTROL", "id":6384300, "ctx":"Checkpointer","msg":"Writing fatal message","attr":{"message":"Got signal: 6 (Aborted).\n"}}
153838:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31380, "ctx":"Checkpointer","msg":"BACKTRACE","attr":{"bt":{"backtrace":[{"a":"55D32A6A4928","b":"55D32285A000","o":"7E4A928","s":"_ZN5mongo15printStackTraceEv","C":"mongo::printStackTrace()","s+":"38"},{"a":"55D32A69F5F7","b":"55D32285A000","o":"7E455F7","s":"abruptQuit","s+":"77"},{"a":"7F133EDA08E0","b":"7F133ED8F000","o":"118E0","s":"funlockfile","s+":"50"},{"a":"7F133EA15CA0","b":"7F133E9E2000","o":"33CA0","s":"gsignal","s+":"110"},{"a":"7F133EA17148","b":"7F133E9E2000","o":"35148","s":"abort","s+":"148"},{"a":"55D327696530","b":"55D32285A000","o":"4E3C530","s":"__wt_abort","s+":"28"},{"a":"55D3276CF0D0","b":"55D32285A000","o":"4E750D0","s":"__hs_delete_reinsert_from_pos.constprop.0.cold","s+":"2D"},{"a":"55D3276CD4A9","b":"55D32285A000","o":"4E734A9","s":"__hs_insert_record","s+":"369"},{"a":"55D3276CEBAD","b":"55D32285A000","o":"4E74BAD","s":"__wt_hs_insert_updates","s+":"12FD"},{"a":"55D3276C576D","b":"55D32285A000","o":"4E6B76D","s":"__rec_write_wrapup","s+":"15D"},{"a":"55D3276C7621","b":"55D32285A000","o":"4E6D621","s":"__wt_reconcile","s+":"511"},{"a":"55D32768670A","b":"55D32285A000","o":"4E2C70A","s":"__wt_sync_file","s+":"53A"},{"a":"55D32776A1E2","b":"55D32285A000","o":"4F101E2","s":"__checkpoint_tree","s+":"102"},{"a":"55D32776B5A2","b":"55D32285A000","o":"4F115A2","s":"__txn_checkpoint_wrapper","s+":"D52"},{"a":"55D32776C866","b":"55D32285A000","o":"4F12866","s":"__wt_txn_checkpoint","s+":"116"},{"a":"55D327715738","b":"55D32285A000","o":"4EBB738","s":"__session_checkpoint.cold","s+":"F1"},{"a":"55D32752947A","b":"55D32285A000","o":"4CCF47A","s":"_ZN5mongo18WiredTigerKVEngine11_checkpointEP12__wt_sessionb","C":"mongo::WiredTigerKVEngine::_checkpoint(__wt_session*, bool)","s+":"FA"},{"a":"55D32752F95D","b":"55D32285A000","o":"4CD595D","s":"_ZN5mongo18WiredTigerKVEngine11_checkpointEPNS_16OperationContextEP12__wt_session","C":"mongo::WiredTigerKVEngine::_checkpoint(mongo::OperationContext*, __wt_session*)","s+":"37D"},{"a":"55D327531D49","b":"55D32285A000","o":"4CD7D49","s":"_ZN5mongo18WiredTigerKVEngine10checkpointEPNS_16OperationContextE","C":"mongo::WiredTigerKVEngine::checkpoint(mongo::OperationContext*)","s+":"49"},{"a":"55D328232159","b":"55D32285A000","o":"59D8159","s":"_ZN5mongo12Checkpointer3runEv","C":"mongo::Checkpointer::run()","s+":"529"},{"a":"55D32A39750A","b":"55D32285A000","o":"7B3D50A","s":"_ZN5mongo13BackgroundJob7jobBodyEv","C":"mongo::BackgroundJob::jobBody()","s+":"31A"},{"a":"55D32A3978EC","b":"55D32285A000","o":"7B3D8EC","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_13BackgroundJob2goEvEUlvE0_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","C":"std::thread::_State_impl<std::thread::_Invoker<std::tuple<mongo::stdx::thread::thread<mongo::BackgroundJob::go()::{lambda()#2}, , 0>(mongo::BackgroundJob::go()::{lambda()#2})::{lambda()#1}> > >::_M_run()","s+":"5C"},{"a":"55D32A8CCCB4","b":"55D32285A000","o":"8072CB4","s":"execute_native_thread_routine","s+":"14"},{"a":"7F133ED9644B","b":"7F133ED8F000","o":"744B","s":"start_thread","s+":"DB"},{"a":"7F133EAD152F","b":"7F133E9E2000","o":"EF52F","s":"clone","s+":"3F"}],"processInfo":{"mongodbVersion":"7.0.12","gitVersion":"b6513ce0781db6818e24619e8a461eae90bc94fc","compiledModules":["enterprise"],"uname":{"sysname":"Linux","release":"5.15.166-111.163.2.mdb.amzn2.x86_64","version":"#1 SMP Thu Sep 19 12:52:20 UTC 2024","machine":"x86_64"},"somap":[{"b":"55D32285A000","elfType":3,"buildId":"D0482F5E1003AC1B"},{"b":"7F133ED8F000","path":"/lib64/libpthread.so.0","elfType":3,"buildId":"0231966C233A89975BC72A53FD9D6BD2792392BD"},{"b":"7F133E9E2000","path":"/lib64/libc.so.6","elfType":3,"buildId":"051FA4AA3B428C4DF43E9D334078061618FF0A7F"}]}}},"tags":[]}
153839:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D32A6A4928","b":"55D32285A000","o":"7E4A928","s":"_ZN5mongo15printStackTraceEv","C":"mongo::printStackTrace()","s+":"38"}}}
153840:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D32A69F5F7","b":"55D32285A000","o":"7E455F7","s":"abruptQuit","s+":"77"}}}
153841:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"7F133EDA08E0","b":"7F133ED8F000","o":"118E0","s":"funlockfile","s+":"50"}}}
153842:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"7F133EA15CA0","b":"7F133E9E2000","o":"33CA0","s":"gsignal","s+":"110"}}}
153843:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"7F133EA17148","b":"7F133E9E2000","o":"35148","s":"abort","s+":"148"}}}
153844:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D327696530","b":"55D32285A000","o":"4E3C530","s":"__wt_abort","s+":"28"}}}
153845:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D3276CF0D0","b":"55D32285A000","o":"4E750D0","s":"__hs_delete_reinsert_from_pos.constprop.0.cold","s+":"2D"}}}
153846:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D3276CD4A9","b":"55D32285A000","o":"4E734A9","s":"__hs_insert_record","s+":"369"}}}
153847:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D3276CEBAD","b":"55D32285A000","o":"4E74BAD","s":"__wt_hs_insert_updates","s+":"12FD"}}}
153848:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D3276C576D","b":"55D32285A000","o":"4E6B76D","s":"__rec_write_wrapup","s+":"15D"}}}
153849:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D3276C7621","b":"55D32285A000","o":"4E6D621","s":"__wt_reconcile","s+":"511"}}}
153850:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D32768670A","b":"55D32285A000","o":"4E2C70A","s":"__wt_sync_file","s+":"53A"}}}
153851:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D32776A1E2","b":"55D32285A000","o":"4F101E2","s":"__checkpoint_tree","s+":"102"}}}
153852:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D32776B5A2","b":"55D32285A000","o":"4F115A2","s":"__txn_checkpoint_wrapper","s+":"D52"}}}
153853:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D32776C866","b":"55D32285A000","o":"4F12866","s":"__wt_txn_checkpoint","s+":"116"}}}
153854:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D327715738","b":"55D32285A000","o":"4EBB738","s":"__session_checkpoint.cold","s+":"F1"}}}
153855:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D32752947A","b":"55D32285A000","o":"4CCF47A","s":"_ZN5mongo18WiredTigerKVEngine11_checkpointEP12__wt_sessionb","C":"mongo::WiredTigerKVEngine::_checkpoint(__wt_session*, bool)","s+":"FA"}}}
153856:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D32752F95D","b":"55D32285A000","o":"4CD595D","s":"_ZN5mongo18WiredTigerKVEngine11_checkpointEPNS_16OperationContextEP12__wt_session","C":"mongo::WiredTigerKVEngine::_checkpoint(mongo::OperationContext*, __wt_session*)","s+":"37D"}}}
153857:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D327531D49","b":"55D32285A000","o":"4CD7D49","s":"_ZN5mongo18WiredTigerKVEngine10checkpointEPNS_16OperationContextE","C":"mongo::WiredTigerKVEngine::checkpoint(mongo::OperationContext*)","s+":"49"}}}
153858:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D328232159","b":"55D32285A000","o":"59D8159","s":"_ZN5mongo12Checkpointer3runEv","C":"mongo::Checkpointer::run()","s+":"529"}}}
153859:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D32A39750A","b":"55D32285A000","o":"7B3D50A","s":"_ZN5mongo13BackgroundJob7jobBodyEv","C":"mongo::BackgroundJob::jobBody()","s+":"31A"}}}
153860:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D32A3978EC","b":"55D32285A000","o":"7B3D8EC","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_13BackgroundJob2goEvEUlvE0_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","C":"std::thread::_State_impl<std::thread::_Invoker<std::tuple<mongo::stdx::thread::thread<mongo::BackgroundJob::go()::{lambda()#2}, , 0>(mongo::BackgroundJob::go()::{lambda()#2})::{lambda()#1}> > >::_M_run()","s+":"5C"}}}
153861:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55D32A8CCCB4","b":"55D32285A000","o":"8072CB4","s":"execute_native_thread_routine","s+":"14"}}}
153862:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"7F133ED9644B","b":"7F133ED8F000","o":"744B","s":"start_thread","s+":"DB"}}}
153863:{"t":{"$date":"2024-10-12T16:27:24.712+00:00"},"s":"I", "c":"CONTROL", "id":31445, "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"7F133EAD152F","b":"7F133E9E2000","o":"EF52F","s":"clone","s+":"3F"}}}
153864:{"t":{"$date":"2024-10-12T16:27:49.738+00:00"},"s":"I", "c":"CONTROL", "id":20698, "ctx":"main","msg":"***** SERVER RESTARTED *****"}