|
Hi,
After upgrade from 4.4.4 to 4.4.9 we get:
{"t":{"$date":"2021-11-06T14:55:01.917+03:00"},"s":"F", "c":"-", "id":23079, "ctx":"ConfigServerCatalogCacheLoader-1","msg":"Invariant failure
|
","attr":{"expr":"chunkVersion >= collectionVersion","file":"src/mongo/s/chunk_manager.cpp","line":614}}
|
{"t":{"$date":"2021-11-06T14:55:01.917+03:00"},"s":"F", "c":"-", "id":23080, "ctx":"ConfigServerCatalogCacheLoader-1","msg":"\n\n***aborting a
|
fter invariant() failure\n\n"}
|
{"t":{"$date":"2021-11-06T14:55:01.917+03:00"},"s":"F", "c":"CONTROL", "id":4757800, "ctx":"ConfigServerCatalogCacheLoader-1","msg":"Writing fatal mes
|
sage","attr":{"message":"Got signal: 6 (Aborted).\n"}}
|
{"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31431, "ctx":"ConfigServerCatalogCacheLoader-1","msg":"BACKTRACE: {bt}",
|
"attr":{"bt":{"backtrace":[{"a":"558F8BAA83AA","b":"558F88CDF000","o":"2DC93AA","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERK
|
NS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1EA"},{"a":"558F8BAA9E39","b":"558F88CDF000","o":"2DCAE39","s":"_ZN5mongo15printStackTraceEv","s
|
+":"29"},{"a":"558F8BAA71C6","b":"558F88CDF000","o":"2DC81C6","s":"_ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo_tPv","s+":"66"},{"a":"7FB72909C6
|
30","b":"7FB72908D000","o":"F630","s":"_L_unlock_13","s+":"34"},{"a":"7FB728CF5387","b":"7FB728CBF000","o":"36387","s":"gsignal","s+":"37"},{"a":"7FB728
|
CF6A78","b":"7FB728CBF000","o":"37A78","s":"abort","s+":"148"},{"a":"558F89C1D089","b":"558F88CDF000","o":"F3E089","s":"_ZN5mongo15invariantFailedEPKcS1
|
_j","s+":"12C"},{"a":"558F89AC7DB2","b":"558F88CDF000","o":"DE8DB2","s":"_ZN5mongo19RoutingTableHistory11makeUpdatedERKSt6vectorINS_9ChunkTypeESaIS2_EE.
|
cold.1193","s+":"130"},{"a":"558F8AA4BD7B","b":"558F88CDF000","o":"1D6CD7B","s":"_ZN5mongo19RoutingTableHistory7makeNewENS_15NamespaceStringEN5boost8opt
|
ionalINS_4UUIDEEENS_10KeyPatternESt10unique_ptrINS_17CollatorInterfaceESt14default_deleteIS8_EEbNS_3OIDERKSt6vectorINS_9ChunkTypeESaISE_EE","s+":"18B"},
|
{"a":"558F8A578136","b":"558F88CDF000","o":"1899136","s":"_ZNSt17_Function_handlerIFvPN5mongo16OperationContextENS0_10StatusWithINS0_18CatalogCacheLoade
|
r26CollectionAndChangedChunksEEEEZNS0_12CatalogCache26_scheduleCollectionRefreshENS0_8WithLockESt10shared_ptrINS8_26CollectionRoutingInfoEntryEERKNS0_15
|
NamespaceStringEiEUlS2_S6_E1_E9_M_invokeERKSt9_Any_dataOS2_OS6_","s+":"296"},{"a":"558F8A55D967","b":"558F88CDF000","o":"187E967","s":"_ZZN5mongo30Confi
|
gServerCatalogCacheLoader14getChunksSinceERKNS_15NamespaceStringENS_12ChunkVersionESt8functionIFvPNS_16OperationContextENS_10StatusWithINS_18CatalogCach
|
eLoader26CollectionAndChangedChunksEEEEEENKUlT_E_clINS_6StatusEEEDaSE_","s+":"167"},{"a":"558F8A55DD77","b":"558F88CDF000","o":"187ED77","s":"_ZZN5mongo
|
15unique_functionIFvNS_6StatusEEE8makeImplIZNS_30ConfigServerCatalogCacheLoader14getChunksSinceERKNS_15NamespaceStringENS_12ChunkVersionESt8functionIFvP
|
NS_16OperationContextENS_10StatusWithINS_18CatalogCacheLoader26CollectionAndChangedChunksEEEEEEUlT_E_EEDaOSJ_EN12SpecificImpl4callEOS1_","s+":"37"},{"a":"558F8B5D4832","b":"558F88CDF000","o":"28F5832","s":"_ZN5mongo10ThreadPool10_doOneTaskEPSt11unique_lockINS_12latch_detail5LatchEE","s+":"132"},{"a":"558F8B5D6E76","b":"558F88CDF000","o":"28F7E76","s":"_ZN5mongo10ThreadPool13_consumeTasksEv","s+":"86"},{"a":"558F8B5D7C21","b":"558F88CDF000","o":"28F8C21","s":"_ZN5mongo10ThreadPool17_workerThreadBodyEPS0_RKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE","s+":"E1"},{"a":"558F8B5D7F50","b":"558F88CDF000","o":"28F8F50","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_10ThreadPool25_startWorkerThread_inlockEvEUlvE2_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"60"},{"a":"558F8BC52D2F","b":"558F88CDF000","o":"2F73D2F","s":"execute_native_thread_routine","s+":"F"},{"a":"7FB729094EA5","b":"7FB72908D000","o":"7EA5","s":"start_thread","s+":"C5"},{"a":"7FB728DBD9FD","b":"7FB728CBF000","o":"FE9FD","s":"clone","s+":"6D"}],"processInfo":{"mongodbVersion":"4.4.9","gitVersion":"b4048e19814bfebac717cf5a880076aa69aba481","compiledModules":[],"uname":{"sysname":"Linux","release":"3.10.0-1160.24.1.el7.x86_64","version":"#1 SMP Thu Apr 8 19:51:47 UTC 2021","machine":"x86_64"},"somap":[{"b":"558F88CDF000","elfType":3,"buildId":"375E0455B64A8CCBA2B20814F34100164730166F"},{"b":"7FB72908D000","path":"/lib64/libpthread.so.0","elfType":3,"buildId":"E10CC8F2B932FC3DAEDA22F8DAC5EBB969524E5B"},{"b":"7FB728CBF000","path":"/lib64/libc.so.6","elfType":3,"buildId":"A317B42B15368ADCAE21C11107691A03EC91059D"}]}}}}
|
{"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8BAA83AA","b":"558F88CDF000","o":"2DC93AA","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1EA"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8BAA9E39","b":"558F88CDF000","o":"2DCAE39","s":"_ZN5mongo15printStackTraceEv","s+":"29"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8BAA71C6","b":"558F88CDF000","o":"2DC81C6","s":"_ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo_tPv","s+":"66"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"7FB72909C630","b":"7FB72908D000","o":"F630","s":"_L_unlock_13","s+":"34"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"7FB728CF5387","b":"7FB728CBF000","o":"36387","s":"gsignal","s+":"37"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"7FB728CF6A78","b":"7FB728CBF000","o":"37A78","s":"abort","s+":"148"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F89C1D089","b":"558F88CDF000","o":"F3E089","s":"_ZN5mongo15invariantFailedEPKcS1_j","s+":"12C"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F89AC7DB2","b":"558F88CDF000","o":"DE8DB2","s":"_ZN5mongo19RoutingTableHistory11makeUpdatedERKSt6vectorINS_9ChunkTypeESaIS2_EE.cold.1193","s+":"130"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8AA4BD7B","b":"558F88CDF000","o":"1D6CD7B","s":"_ZN5mongo19RoutingTableHistory7makeNewENS_15NamespaceStringEN5boost8optionalINS_4UUIDEEENS_10KeyPatternESt10unique_ptrINS_17CollatorInterfaceESt14default_deleteIS8_EEbNS_3OIDERKSt6vectorINS_9ChunkTypeESaISE_EE","s+":"18B"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8A578136","b":"558F88CDF000","o":"1899136","s":"_ZNSt17_Function_handlerIFvPN5mongo16OperationContextENS0_10StatusWithINS0_18CatalogCacheLoader26CollectionAndChangedChunksEEEEZNS0_12CatalogCache26_scheduleCollectionRefreshENS0_8WithLockESt10shared_ptrINS8_26CollectionRoutingInfoEntryEERKNS0_15NamespaceStringEiEUlS2_S6_E1_E9_M_invokeERKSt9_Any_dataOS2_OS6_","s+":"296"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8A55D967","b":"558F88CDF000","o":"187E967","s":"_ZZN5mongo30ConfigServerCatalogCacheLoader14getChunksSinceERKNS_15NamespaceStringENS_12ChunkVersionESt8functionIFvPNS_16OperationContextENS_10StatusWithINS_18CatalogCacheLoader26CollectionAndChangedChunksEEEEEENKUlT_E_clINS_6StatusEEEDaSE_","s+":"167"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8A55DD77","b":"558F88CDF000","o":"187ED77","s":"_ZZN5mongo15unique_functionIFvNS_6StatusEEE8makeImplIZNS_30ConfigServerCatalogCacheLoader14getChunksSinceERKNS_15NamespaceStringENS_12ChunkVersionESt8functionIFvPNS_16OperationContextENS_10StatusWithINS_18CatalogCacheLoader26CollectionAndChangedChunksEEEEEEUlT_E_EEDaOSJ_EN12SpecificImpl4callEOS1_","s+":"37"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8B5D4832","b":"558F88CDF000","o":"28F5832","s":"_ZN5mongo10ThreadPool10_doOneTaskEPSt11unique_lockINS_12latch_detail5LatchEE","s+":"132"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8B5D6E76","b":"558F88CDF000","o":"28F7E76","s":"_ZN5mongo10ThreadPool13_consumeTasksEv","s+":"86"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.068+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8B5D7C21","b":"558F88CDF000","o":"28F8C21","s":"_ZN5mongo10ThreadPool17_workerThreadBodyEPS0_RKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE","s+":"E1"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.068+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8B5D7F50","b":"558F88CDF000","o":"28F8F50","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_10ThreadPool25_startWorkerThread_inlockEvEUlvE2_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"60"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.068+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8BC52D2F","b":"558F88CDF000","o":"2F73D2F","s":"execute_native_thread_routine","s+":"F"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.068+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"7FB729094EA5","b":"7FB72908D000","o":"7EA5","s":"start_thread","s+":"C5"}}}
|
{"t":{"$date":"2021-11-06T14:55:02.068+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"7FB728DBD9FD","b":"7FB728CBF000","o":"FE9FD","s":"clone","s+":"6D"}}}
|
|
Downgrading does not solve the problem.
What can be done?
|