Uploaded image for project: 'WiredTiger'
  1. WiredTiger
  2. WT-12090

MongoDB 7.0 Crashes often with Invariant Failure

    • Type: Icon: Bug Bug
    • Resolution: Unresolved
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: 6.0.9, 7.0.0
    • Component/s: None
    • StorEng - Defined Pipeline

      I am running mongodb with Docker Compose and the service definition looks like this

          db.selfmade.ninja:
              container_name: "db.selfmade.ninja"
              image: "mongo:latest"
              restart: "unless-stopped"
              ports:
                  - "127.0.0.1:27017:27017"
              volumes:
                  - "mongodb:/data/db"
              networks:
                  - backend
              labels:
                  - "traefik.enable=false"
      

      I updated to MongoDB 7.0 recently and before that I was on 6.0. But I felt this issues were there in 6.0 also. 

      The server is running good for a while, them this happens 2-3 times every hour. If I restart the server it is good for sometime. But then this error happens and server crashes. Then the container restarts.

      What could be the root cause of this error ?

      {"t":\{"$date":"2023-09-04T06:22:05.513+00:00"},"s":"I",  "c":"NETWORK",  "id":22943,   "ctx":"listener","msg":"Connection accepted","attr":\{"remote":"10.100.0.13:56080","uuid":{"uuid":{"$uuid":"09c2c7ca-63b9-491b-a006-45c57e8a057d"}},"connectionId":375,"connectionCount":32}}
      {"t":\{"$date":"2023-09-04T06:22:05.513+00:00"},"s":"I",  "c":"NETWORK",  "id":51800,   "ctx":"conn375","msg":"client metadata","attr":\{"remote":"10.100.0.13:56080","client":"conn375","doc":{"driver":{"name":"mongoc / ext-mongodb:PHP / PHPLIB ","version":"1.24.1 / 1.16.1 / 1.16.0 "},"os":\{"type":"Linux","name":"Ubuntu","version":"22.04","architecture":"x86_64"},"platform":"PHP 8.1.2-1ubuntu2.13 cfg=0x03515620c9 posix=200809 stdc=201710 CC=GCC 11.3.0 CFLAGS=\"\" LDFLAGS=\"\""}}}
      {"t":\{"$date":"2023-09-04T06:22:05.514+00:00"},"s":"I",  "c":"NETWORK",  "id":6788700, "ctx":"conn375","msg":"Received first command on ingress connection since session start or auth handshake","attr":\{"elapsedMillis":0}}
      {"t":\{"$date":"2023-09-04T06:22:05.569+00:00"},"s":"I",  "c":"NETWORK",  "id":22944,   "ctx":"conn374","msg":"Connection ended","attr":\{"remote":"10.100.0.13:56078","uuid":{"uuid":{"$uuid":"c309384d-51fe-4dd5-8ca5-755e40098aaf"}},"connectionId":374,"connectionCount":31}}
      {"t":\{"$date":"2023-09-04T06:22:05.837+00:00"},"s":"I",  "c":"NETWORK",  "id":22943,   "ctx":"listener","msg":"Connection accepted","attr":\{"remote":"10.100.0.13:56092","uuid":{"uuid":{"$uuid":"ff59fcb5-221e-4b2c-bc6b-47d0b5fcd98e"}},"connectionId":376,"connectionCount":32}}
      {"t":\{"$date":"2023-09-04T06:22:05.837+00:00"},"s":"I",  "c":"NETWORK",  "id":51800,   "ctx":"conn376","msg":"client metadata","attr":\{"remote":"10.100.0.13:56092","client":"conn376","doc":{"driver":{"name":"mongoc / ext-mongodb:PHP / PHPLIB ","version":"1.24.1 / 1.16.1 / 1.16.0 "},"os":\{"type":"Linux","name":"Ubuntu","version":"22.04","architecture":"x86_64"},"platform":"PHP 8.1.2-1ubuntu2.13 cfg=0x03515620c9 posix=200809 stdc=201710 CC=GCC 11.3.0 CFLAGS=\"\" LDFLAGS=\"\""}}}
      {"t":\{"$date":"2023-09-04T06:22:05.840+00:00"},"s":"I",  "c":"NETWORK",  "id":6788700, "ctx":"conn376","msg":"Received first command on ingress connection since session start or auth handshake","attr":\{"elapsedMillis":2}}
      {"t":\{"$date":"2023-09-04T06:22:05.892+00:00"},"s":"I",  "c":"NETWORK",  "id":22944,   "ctx":"conn376","msg":"Connection ended","attr":\{"remote":"10.100.0.13:56092","uuid":{"uuid":{"$uuid":"ff59fcb5-221e-4b2c-bc6b-47d0b5fcd98e"}},"connectionId":376,"connectionCount":31}}
      {"t":\{"$date":"2023-09-04T06:22:05.892+00:00"},"s":"I",  "c":"NETWORK",  "id":22944,   "ctx":"conn375","msg":"Connection ended","attr":\{"remote":"10.100.0.13:56080","uuid":{"uuid":{"$uuid":"09c2c7ca-63b9-491b-a006-45c57e8a057d"}},"connectionId":375,"connectionCount":30}}
      {"t":\{"$date":"2023-09-04T06:22:07.788+00:00"},"s":"I",  "c":"COMMAND",  "id":51803,   "ctx":"conn34","msg":"Slow query","attr":\{"type":"command","ns":"selfmadelabs.$cmd","command":{"update":"labstats","ordered":true,"lsid":{"id":{"$uuid":"fa5453b3-980c-4e17-a8c6-46ae49513c04"}},"$db":"selfmadelabs"},"numYields":1372,"reslen":60,"locks":\{"ParallelBatchWriterMode":{"acquireCount":{"r":5096}},"FeatureCompatibilityVersion":\{"acquireCount":{"w":5096}},"ReplicationStateTransition":\{"acquireCount":{"w":5096}},"Global":\{"acquireCount":{"w":5096}},"Database":\{"acquireCount":{"w":5096}},"Collection":\{"acquireCount":{"w":5096}}},"flowControl":\{"acquireCount":5096},"storage":{},"cpuNanos":3069647525,"remote":"10.100.0.12:59344","protocol":"op_msg","durationMillis":3115}}
      {"t":\{"$date":"2023-09-04T06:22:10.145+00:00"},"s":"E",  "c":"WT",       "id":22435,   "ctx":"conn27","msg":"WiredTiger error message","attr":\{"error":22,"message":{"ts_sec":1693808530,"ts_usec":145725,"thread":"1:0x7f2df5821640","session_name":"WT_SESSION.get_value","category":"WT_VERB_DEFAULT","category_id":9,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"__wt_txn_context_prepare_check:19:not permitted in a prepared transaction","error_str":"Invalid argument","error_code":22}}}
      {"t":\{"$date":"2023-09-04T06:22:10.145+00:00"},"s":"F",  "c":"ASSERT",   "id":23083,   "ctx":"conn27","msg":"Invariant failure","attr":\{"expr":"c->get_value(c, &value)","error":"BadValue: 22: Invalid argument","file":"src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp","line":1975}}
      {"t":\{"$date":"2023-09-04T06:22:10.145+00:00"},"s":"F",  "c":"ASSERT",   "id":23084,   "ctx":"conn27","msg":"\n\n***aborting after invariant() failure\n\n"}
      {"t":\{"$date":"2023-09-04T06:22:10.145+00:00"},"s":"F",  "c":"CONTROL",  "id":6384300, "ctx":"conn27","msg":"Writing fatal message","attr":\{"message":"\n"}}
      {"t":\{"$date":"2023-09-04T06:22:10.145+00:00"},"s":"F",  "c":"CONTROL",  "id":6384300, "ctx":"conn27","msg":"Writing fatal message","attr":\{"message":"Got signal: 6 (Aborted).\n"}}
      {"t":\{"$date":"2023-09-04T06:22:10.280+00:00"},"s":"I",  "c":"CONTROL",  "id":31380,   "ctx":"conn27","msg":"BACKTRACE","attr":\{"bt":{"backtrace":[{"a":"5633C7E1AFB4","b":"5633C03FE000","o":"7A1CFB4","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_117getStackTraceImplERKNS1_7OptionsE.constprop.0","C":"mongo::stack_trace_detail::(anonymous namespace)::getStackTraceImpl(mongo::stack_trace_detail::(anonymous namespace)::Options const&) [clone .constprop.0]","s+":"224"},\{"a":"5633C7E1CC68","b":"5633C03FE000","o":"7A1EC68","s":"_ZN5mongo15printStackTraceEv","C":"mongo::printStackTrace()","s+":"38"},\{"a":"5633C7E177EA","b":"5633C03FE000","o":"7A197EA","s":"abruptQuit","s+":"6A"},\{"a":"7F2E09AA8520","b":"7F2E09A66000","o":"42520","s":"__sigaction","s+":"50"},\{"a":"7F2E09AFCA7C","b":"7F2E09A66000","o":"96A7C","s":"pthread_kill","s+":"12C"},\{"a":"7F2E09AA8476","b":"7F2E09A66000","o":"42476","s":"raise","s+":"16"},\{"a":"7F2E09A8E7F3","b":"7F2E09A66000","o":"287F3","s":"abort","s+":"D3"},\{"a":"5633C7E09A47","b":"5633C03FE000","o":"7A0BA47","s":"_ZN5mongo12_GLOBAL__N_19callAbortEv","C":"mongo::(anonymous namespace)::callAbort()","s+":"1B"},\{"a":"5633C7E0AF98","b":"5633C03FE000","o":"7A0CF98","s":"_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j","C":"mongo::invariantOKFailed(char const*, mongo::Status const&, char const*, unsigned int)","s+":"279"},\{"a":"5633C4D159D5","b":"5633C03FE000","o":"49179D5","s":"_ZN5mongo31WiredTigerRecordStoreCursorBase4nextEv.cold","C":"mongo::WiredTigerRecordStoreCursorBase::next() [clone .cold]","s+":"2BB"},\{"a":"5633C5D16C51","b":"5633C03FE000","o":"5918C51","s":"_ZN5mongo3sbe9ScanStage7getNextEv","C":"mongo::sbe::ScanStage::getNext()","s+":"2D1"},\{"a":"5633C58A81E7","b":"5633C03FE000","o":"54AA1E7","s":"_ZN5mongo3sbe11FilterStageILb0ELb0EE7getNextEv","C":"mongo::sbe::FilterStage<false, false>::getNext()","s+":"57"},\{"a":"5633C5C23915","b":"5633C03FE000","o":"5825915","s":"_ZN5mongo3sbe14LimitSkipStage7getNextEv","C":"mongo::sbe::LimitSkipStage::getNext()","s+":"D5"},\{"a":"5633C584E2EC","b":"5633C03FE000","o":"54502EC","s":"_ZN5mongo13fetchNextImplINS_7BSONObjEEENS_3sbe9PlanStateEPNS2_9PlanStageEPNS2_5value12SlotAccessorES8_PT_PNS_8RecordIdEb","C":"mongo::sbe::PlanState mongo::fetchNextImpl<mongo::BSONObj>(mongo::sbe::PlanStage*, mongo::sbe::value::SlotAccessor*, mongo::sbe::value::SlotAccessor*, mongo::BSONObj*, mongo::RecordId*, bool)","s+":"4C"},\{"a":"5633C584F62C","b":"5633C03FE000","o":"545162C","s":"_ZN5mongo15PlanExecutorSBE11getNextImplINS_7BSONObjEEENS_12PlanExecutor9ExecStateEPT_PNS_8RecordIdE","C":"mongo::PlanExecutor::ExecState mongo::PlanExecutorSBE::getNextImpl<mongo::BSONObj>(mongo::BSONObj*, mongo::RecordId*)","s+":"38C"},\{"a":"5633C584A98B","b":"5633C03FE000","o":"544C98B","s":"_ZN5mongo15PlanExecutorSBE7getNextEPNS_7BSONObjEPNS_8RecordIdE","C":"mongo::PlanExecutorSBE::getNext(mongo::BSONObj*, mongo::RecordId*)","s+":"5B"},\{"a":"5633C50322E0","b":"5633C03FE000","o":"4C342E0","s":"_ZN5mongo12_GLOBAL__N_17FindCmd10Invocation3runEPNS_16OperationContextEPNS_3rpc21ReplyBuilderInterfaceE","C":"mongo::(anonymous namespace)::FindCmd::Invocation::run(mongo::OperationContext*, mongo::rpc::ReplyBuilderInterface*)","s+":"E70"},\{"a":"5633C747D9B0","b":"5633C03FE000","o":"707F9B0","s":"_ZN5mongo14CommandHelpers20runCommandInvocationEPNS_16OperationContextERKNS_12OpMsgRequestEPNS_17CommandInvocationEPNS_3rpc21ReplyBuilderInterfaceE","C":"mongo::CommandHelpers::runCommandInvocation(mongo::OperationContext*, mongo::OpMsgRequest const&, mongo::CommandInvocation*, mongo::rpc::ReplyBuilderInterface*)","s+":"60"},\{"a":"5633C74818ED","b":"5633C03FE000","o":"70838ED","s":"_ZN5mongo14CommandHelpers20runCommandInvocationESt10shared_ptrINS_23RequestExecutionContextEES1_INS_17CommandInvocationEEb","C":"mongo::CommandHelpers::runCommandInvocation(std::shared_ptr<mongo::RequestExecutionContext>, std::shared_ptr<mongo::CommandInvocation>, bool)","s+":"CD"},\{"a":"5633C3E8D3F0","b":"5633C03FE000","o":"3A8F3F0","s":"_ZN5mongo12_GLOBAL__N_120runCommandInvocationESt10shared_ptrINS_23RequestExecutionContextEES1_INS_17CommandInvocationEE","C":"mongo::(anonymous namespace)::runCommandInvocation(std::shared_ptr<mongo::RequestExecutionContext>, std::shared_ptr<mongo::CommandInvocation>)","s+":"B0"},\{"a":"5633C3E8F996","b":"5633C03FE000","o":"3A91996","s":"_ZN5mongo12_GLOBAL__N_113InvokeCommand3runEv","C":"mongo::(anonymous namespace)::InvokeCommand::run()","s+":"236"},\{"a":"5633C3E97311","b":"5633C03FE000","o":"3A99311","s":"_ZN5mongo12_GLOBAL__N_114RunCommandImpl11_runCommandEv","C":"mongo::(anonymous namespace)::RunCommandImpl::_runCommand()","s+":"2A1"},\{"a":"5633C3E98DA6","b":"5633C03FE000","o":"3A9ADA6","s":"_ZN5mongo12_GLOBAL__N_114RunCommandImpl8_runImplEv","C":"mongo::(anonymous namespace)::RunCommandImpl::_runImpl()","s+":"96"},\{"a":"5633C3E91736","b":"5633C03FE000","o":"3A93736","s":"_ZN5mongo12_GLOBAL__N_114RunCommandImpl3runEv","C":"mongo::(anonymous namespace)::RunCommandImpl::run()","s+":"136"},\{"a":"5633C3E9A8F9","b":"5633C03FE000","o":"3A9C8F9","s":"_ZN5mongo12_GLOBAL__N_119ExecCommandDatabase12_commandExecEv","C":"mongo::(anonymous namespace)::ExecCommandDatabase::_commandExec()","s+":"1E9"},\{"a":"5633C3EA0918","b":"5633C03FE000","o":"3AA2918","s":"_ZN5mongo19makeReadyFutureWithIZNOS_11future_util10AsyncStateINS_12_GLOBAL__N_119ExecCommandDatabaseEE13thenWithStateIZZNS3_14executeCommandESt10shared_ptrINS3_13HandleRequest16ExecutionContextEEENUlvE0_clEvEUlPT_E_EEDaOSC_EUlvE_EENS_6FutureINS_14future_details17UnwrappedTypeImplINSt13invoke_resultISF_JEE4typeEE4typeEEESF_","s+":"48"},\{"a":"5633C3EA122C","b":"5633C03FE000","o":"3AA322C","s":"_ZZN5mongo15unique_functionIFvPNS_14future_details15SharedStateBaseEEE8makeImplIZNS1_10FutureImplINS1_8FakeVoidEE16makeContinuationIvZZNOS9_4thenINS_19CleanupFuturePolicyILb0EEEZNS_12_GLOBAL__N_114executeCommandESt10shared_ptrINSE_13HandleRequest16ExecutionContextEEEUlvE0_EEDaT_OT0_ENKUlvE1_clEvEUlPNS1_15SharedStateImplIS8_EESQ_E_EENS7_ISK_EESM_EUlS3_E_EEDaOSK_EN12SpecificImpl4callEOS3_","C":"mongo::unique_function<void (mongo::future_details::SharedStateBase*)>::makeImpl<mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::makeContinuation<void, mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::then<mongo::CleanupFuturePolicy<false>, mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::{lambda()#2}>(mongo::CleanupFuturePolicy<false>, mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::\{lambda()#2}&&) &&::\{lambda()#3}::operator()() const::\{lambda(mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*, mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*)#1}>(mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::\{lambda()#2}&&)::\{lambda(mongo::future_details::SharedStateBase*)#1}>(mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::makeContinuation<void, mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::then<mongo::CleanupFuturePolicy<false>, mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::\{lambda()#2}>(mongo::CleanupFuturePolicy<false>, mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::\{lambda()#2}&&) &&::\{lambda()#3}::operator()() const::\{lambda(mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*, mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*)#1}>(mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::\{lambda()#2}&&)::\{lambda(mongo::future_details::SharedStateBase*)#1}&&)::SpecificImpl::call(mongo::future_details::SharedStateBase*&&)","s+":"1FC"},\{"a":"5633C3E37B27","b":"5633C03FE000","o":"3A39B27","s":"_ZN5mongo14future_details15SharedStateBase20transitionToFinishedEv","C":"mongo::future_details::SharedStateBase::transitionToFinished()","s+":"107"},\{"a":"5633C3EABD7C","b":"5633C03FE000","o":"3AADD7C","s":"_ZNO5mongo14future_details10FutureImplINS0_8FakeVoidEE17propagateResultToEPNS0_15SharedStateImplIS2_EE","C":"mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::propagateResultTo(mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*) &&","s+":"1AC"},\{"a":"5633C3E999A3","b":"5633C03FE000","o":"3A9B9A3","s":"_ZZN5mongo15unique_functionIFvPNS_14future_details15SharedStateBaseEEE8makeImplIZNS1_10FutureImplINS1_8FakeVoidEE16makeContinuationIvZZNOS9_4thenINS_19CleanupFuturePolicyILb0EEEZNS_12_GLOBAL__N_114executeCommandESt10shared_ptrINSE_13HandleRequest16ExecutionContextEEEUlvE_EEDaT_OT0_ENKUlvE1_clEvEUlPNS1_15SharedStateImplIS8_EESQ_E_EENS7_ISK_EESM_EUlS3_E_EEDaOSK_EN12SpecificImpl4callEOS3_","C":"mongo::unique_function<void (mongo::future_details::SharedStateBase*)>::makeImpl<mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::makeContinuation<void, mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::then<mongo::CleanupFuturePolicy<false>, mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::{lambda()#1}>(mongo::CleanupFuturePolicy<false>, mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::\{lambda()#1}&&) &&::\{lambda()#3}::operator()() const::\{lambda(mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*, mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*)#1}>(mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::\{lambda()#1}&&)::\{lambda(mongo::future_details::SharedStateBase*)#1}>(mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::makeContinuation<void, mongo::future_details::FutureImpl<mongo::future_details::FakeVoid>::then<mongo::CleanupFuturePolicy<false>, mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::\{lambda()#1}>(mongo::CleanupFuturePolicy<false>, mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::\{lambda()#1}&&) &&::\{lambda()#3}::operator()() const::\{lambda(mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*, mongo::future_details::SharedStateImpl<mongo::future_details::FakeVoid>*)#1}>(mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)::\{lambda()#1}&&)::\{lambda(mongo::future_details::SharedStateBase*)#1}&&)::SpecificImpl::call(mongo::future_details::SharedStateBase*&&)","s+":"93"},\{"a":"5633C3E37B27","b":"5633C03FE000","o":"3A39B27","s":"_ZN5mongo14future_details15SharedStateBase20transitionToFinishedEv","C":"mongo::future_details::SharedStateBase::transitionToFinished()","s+":"107"},\{"a":"5633C3EA18D1","b":"5633C03FE000","o":"3AA38D1","s":"_ZN5mongo12_GLOBAL__N_114executeCommandESt10shared_ptrINS0_13HandleRequest16ExecutionContextEE","C":"mongo::(anonymous namespace)::executeCommand(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)","s+":"641"},\{"a":"5633C3EA238B","b":"5633C03FE000","o":"3AA438B","s":"_ZN5mongo12_GLOBAL__N_116receivedCommandsESt10shared_ptrINS0_13HandleRequest16ExecutionContextEE","C":"mongo::(anonymous namespace)::receivedCommands(std::shared_ptr<mongo::(anonymous namespace)::HandleRequest::ExecutionContext>)","s+":"43B"},\{"a":"5633C3EA2E78","b":"5633C03FE000","o":"3AA4E78","s":"_ZN5mongo12_GLOBAL__N_115CommandOpRunner3runEv","C":"mongo::(anonymous namespace)::CommandOpRunner::run()","s+":"48"},\{"a":"5633C3E9580C","b":"5633C03FE000","o":"3A9780C","s":"_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageESt10unique_ptrIKNS0_5HooksESt14default_deleteIS8_EE","C":"mongo::ServiceEntryPointCommon::handleRequest(mongo::OperationContext*, mongo::Message const&, std::unique_ptr<mongo::ServiceEntryPointCommon::Hooks const, std::default_delete<mongo::ServiceEntryPointCommon::Hooks const> >)","s+":"37C"},\{"a":"5633C3E878E0","b":"5633C03FE000","o":"3A898E0","s":"_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE","C":"mongo::ServiceEntryPointMongod::handleRequest(mongo::OperationContext*, mongo::Message const&)","s+":"50"},\{"a":"5633C537E604","b":"5633C03FE000","o":"4F80604","s":"_ZN5mongo9transport15SessionWorkflow4Impl13_dispatchWorkEv","C":"mongo::transport::SessionWorkflow::Impl::_dispatchWork()","s+":"144"},\{"a":"5633C537EE57","b":"5633C03FE000","o":"4F80E57","s":"_ZZNO5mongo14future_details10FutureImplISt10unique_ptrINS_9transport15SessionWorkflow4Impl8WorkItemESt14default_deleteIS6_EEE4thenINS_19CleanupFuturePolicyILb0EEEZNS5_15_doOneIterationEvEUlT_E_EEDaSE_OT0_ENKUlOS9_E_clESI_.isra.0","C":"mongo::future_details::FutureImpl<std::unique_ptr<mongo::transport::SessionWorkflow::Impl::WorkItem, std::default_delete<mongo::transport::SessionWorkflow::Impl::WorkItem> > >::then<mongo::CleanupFuturePolicy<false>, mongo::transport::SessionWorkflow::Impl::_doOneIteration()::{lambda(auto:1)#1}>(mongo::CleanupFuturePolicy<false>, mongo::transport::SessionWorkflow::Impl::_doOneIteration()::\{lambda(auto:1)#1}&&) &&::\{lambda(std::unique_ptr<mongo::transport::SessionWorkflow::Impl::WorkItem, std::default_delete<mongo::transport::SessionWorkflow::Impl::WorkItem> >&&)#1}::operator()(std::unique_ptr<mongo::transport::SessionWorkflow::Impl::WorkItem, std::default_delete<mongo::transport::SessionWorkflow::Impl::WorkItem> >&&) const [clone .isra.0]","s+":"47"},\{"a":"5633C53806E5","b":"5633C03FE000","o":"4F826E5","s":"_ZN5mongo9transport15SessionWorkflow4Impl15_doOneIterationEv","C":"mongo::transport::SessionWorkflow::Impl::_doOneIteration()","s+":"535"},\{"a":"5633C5380FCD","b":"5633C03FE000","o":"4F82FCD","s":"_ZZN5mongo15unique_functionIFvNS_6StatusEEE8makeImplIZNS_9transport15SessionWorkflow4Impl18_scheduleIterationEvEUlS1_E_EEDaOT_EN12SpecificImpl4callEOS1_","C":"mongo::unique_function<void (mongo::Status)>::makeImpl<mongo::transport::SessionWorkflow::Impl::_scheduleIteration()::{lambda(mongo::Status)#1}>(mongo::transport::SessionWorkflow::Impl::_scheduleIteration()::\{lambda(mongo::Status)#1}&&)::SpecificImpl::call(mongo::Status&&)","s+":"5D"},\{"a":"5633C5382E24","b":"5633C03FE000","o":"4F84E24","s":"_ZZN5mongo15unique_functionIFvNS_6StatusEEE8makeImplIZNS_9transport15SessionWorkflow4Impl15_captureContextES3_EUlS1_E_EEDaOT_EN12SpecificImpl4callEOS1_","C":"mongo::unique_function<void (mongo::Status)>::makeImpl<mongo::transport::SessionWorkflow::Impl::_captureContext(mongo::unique_function<void (mongo::Status)>)::{lambda(mongo::Status)#1}>(mongo::transport::SessionWorkflow::Impl::_captureContext(mongo::unique_function<void (mongo::Status)>)::\{lambda(mongo::Status)#1}&&)::SpecificImpl::call(mongo::Status&&)","s+":"94"},\{"a":"5633C74D5C24","b":"5633C03FE000","o":"70D7C24","s":"_ZZN5mongo15unique_functionIFvvEE8makeImplIZNS_9transport26ServiceExecutorSynchronous11SharedState8scheduleENS0_IFvNS_6StatusEEEEEUlvE0_EEDaOT_EN12SpecificImpl4callEv","C":"mongo::unique_function<void ()>::makeImpl<mongo::transport::ServiceExecutorSynchronous::SharedState::schedule(mongo::unique_function<void (mongo::Status)>)::{lambda()#2}>(mongo::transport::ServiceExecutorSynchronous::SharedState::schedule(mongo::unique_function<void (mongo::Status)>)::\{lambda()#2}&&)::SpecificImpl::call()","s+":"C4"},\{"a":"5633C74D8CDD","b":"5633C03FE000","o":"70DACDD","s":"_ZN5mongo9transport12_GLOBAL__N_17runFuncEPv","C":"mongo::transport::(anonymous namespace)::runFunc(void*)","s+":"3DD"},\{"a":"7F2E09AFAB43","b":"7F2E09A66000","o":"94B43","s":"pthread_condattr_setpshared","s+":"513"},\{"a":"7F2E09B8BBB4","b":"7F2E09A66000","o":"125BB4","s":"clone","s+":"44"}],"processInfo":\{"mongodbVersion":"7.0.0","gitVersion":"37d84072b5c5b9fd723db5fa133fb202ad2317f1","compiledModules":[],"uname":{"sysname":"Linux","release":"5.15.0-82-generic","version":"#91-Ubuntu SMP Mon Aug 14 14:14:14 UTC 2023","machine":"x86_64"},"somap":[\{"b":"5633C03FE000","elfType":3,"buildId":"36667979E9E7865B"},\{"b":"7F2E09A66000","path":"/lib/x86_64-linux-gnu/libc.so.6","elfType":3,"buildId":"69389D485A9793DBE873F0EA2C93E02EFAA9AA3D"}]}}},"tags":[]}
      

      What could be the root cause of this error ?

      In another cause, I was able to trace back the connection context to a connection opened 20 mins ago.

      db.selfmade.ninja | {"t":{"$date":"2023-09-04T07:54:55.867+00:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn214","msg":"client metadata","attr":{"remote":"172.18.0.1:38568","client":"conn214","doc":{"driver":{"name":"mongoc / ext-mongodb:PHP / PHPLIB ","version":"1.24.1 / 1.16.1 / 1.16.0 "},"os":{"type":"Linux","name":"Ubuntu","version":"22.04","architecture":"x86_64"},"platform":"PHP 8.1.2-1ubuntu2.13 cfg=0x03515620c9 posix=200809 stdc=201710 CC=GCC 11.3.0 CFLAGS=\"\" LDFLAGS=\"\""}}}
      

      the context connection conn214 threw the error after over 20 mins from its first occurrence in log and then after 20 mins, it threw this followed by the stacktrace.

      db.selfmade.ninja | {"t":{"$date":"2023-09-04T08:13:20.370+00:00"},"s":"E", "c":"WT", "id":22435, "ctx":"conn214","msg":"WiredTiger error message","attr":{"error":22,"message":{"ts_sec":1693815200,"ts_usec":370599,"thread":"1:0x7f380caf0640","session_name":"WT_SESSION.get_value","category":"WT_VERB_DEFAULT","category_id":9,"verbose_level":"ERROR","verbose_level_id":-3,"msg":"__wt_txn_context_prepare_check:19:not permitted in a prepared transaction","error_str":"Invalid argument","error_code":22}}} db.selfmade.ninja | {"t":{"$date":"2023-09-04T08:13:20.370+00:00"},"s":"F", "c":"ASSERT", "id":23083, "ctx":"conn214","msg":"Invariant failure","attr":{"expr":"c->get_value(c, &value)","error":"BadValue: 22: Invalid argument","file":"src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp","line":1975}} db.selfmade.ninja | {"t":{"$date":"2023-09-04T08:13:20.370+00:00"},"s":"F", "c":"ASSERT", "id":23084, "ctx":"conn214","msg":"\n\n***aborting after invariant() failure\n\n"} db.selfmade.ninja | {"t":{"$date":"2023-09-04T08:13:20.370+00:00"},"s":"F", "c":"CONTROL", "id":6384300, "ctx":"conn214","msg":"Writing fatal message","attr":{"message":"\n"}} db.selfmade.ninja | {"t":{"$date":"2023-09-04T08:13:20.370+00:00"},"s":"F", "c":"CONTROL", "id":6384300, "ctx":"conn214","msg":"Writing fatal message","attr":{"message":"Got signal: 6 (Aborted).\n"}}
      

        1. mongo-repair-log.json
          19 kB
        2. mongolog2
          3.84 MB
        3. mongolog1
          3.69 MB
        4. mongolog
          3.52 MB

            Assignee:
            backlog-server-storage-engines [DO NOT USE] Backlog - Storage Engines Team
            Reporter:
            sibi.nandhu@gmail.com Sibidharan Nandakumar
            Votes:
            0 Vote for this issue
            Watchers:
            11 Start watching this issue

              Created:
              Updated: