-
Type: Bug
-
Resolution: Done
-
Priority: Major - P3
-
None
-
Affects Version/s: None
-
Component/s: None
-
None
-
Sharding EMEA
-
ALL
-
Sharding EMEA 2021-12-13
Hi,
After upgrade from 4.4.4 to 4.4.9 we get:
{"t":{"$date":"2021-11-06T14:55:01.917+03:00"},"s":"F", "c":"-", "id":23079, "ctx":"ConfigServerCatalogCacheLoader-1","msg":"Invariant failure ","attr":{"expr":"chunkVersion >= collectionVersion","file":"src/mongo/s/chunk_manager.cpp","line":614}} {"t":{"$date":"2021-11-06T14:55:01.917+03:00"},"s":"F", "c":"-", "id":23080, "ctx":"ConfigServerCatalogCacheLoader-1","msg":"\n\n***aborting a fter invariant() failure\n\n"} {"t":{"$date":"2021-11-06T14:55:01.917+03:00"},"s":"F", "c":"CONTROL", "id":4757800, "ctx":"ConfigServerCatalogCacheLoader-1","msg":"Writing fatal mes sage","attr":{"message":"Got signal: 6 (Aborted).\n"}} {"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31431, "ctx":"ConfigServerCatalogCacheLoader-1","msg":"BACKTRACE: {bt}", "attr":{"bt":{"backtrace":[{"a":"558F8BAA83AA","b":"558F88CDF000","o":"2DC93AA","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERK NS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1EA"},{"a":"558F8BAA9E39","b":"558F88CDF000","o":"2DCAE39","s":"_ZN5mongo15printStackTraceEv","s +":"29"},{"a":"558F8BAA71C6","b":"558F88CDF000","o":"2DC81C6","s":"_ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo_tPv","s+":"66"},{"a":"7FB72909C6 30","b":"7FB72908D000","o":"F630","s":"_L_unlock_13","s+":"34"},{"a":"7FB728CF5387","b":"7FB728CBF000","o":"36387","s":"gsignal","s+":"37"},{"a":"7FB728 CF6A78","b":"7FB728CBF000","o":"37A78","s":"abort","s+":"148"},{"a":"558F89C1D089","b":"558F88CDF000","o":"F3E089","s":"_ZN5mongo15invariantFailedEPKcS1 _j","s+":"12C"},{"a":"558F89AC7DB2","b":"558F88CDF000","o":"DE8DB2","s":"_ZN5mongo19RoutingTableHistory11makeUpdatedERKSt6vectorINS_9ChunkTypeESaIS2_EE. cold.1193","s+":"130"},{"a":"558F8AA4BD7B","b":"558F88CDF000","o":"1D6CD7B","s":"_ZN5mongo19RoutingTableHistory7makeNewENS_15NamespaceStringEN5boost8opt ionalINS_4UUIDEEENS_10KeyPatternESt10unique_ptrINS_17CollatorInterfaceESt14default_deleteIS8_EEbNS_3OIDERKSt6vectorINS_9ChunkTypeESaISE_EE","s+":"18B"}, {"a":"558F8A578136","b":"558F88CDF000","o":"1899136","s":"_ZNSt17_Function_handlerIFvPN5mongo16OperationContextENS0_10StatusWithINS0_18CatalogCacheLoade r26CollectionAndChangedChunksEEEEZNS0_12CatalogCache26_scheduleCollectionRefreshENS0_8WithLockESt10shared_ptrINS8_26CollectionRoutingInfoEntryEERKNS0_15 NamespaceStringEiEUlS2_S6_E1_E9_M_invokeERKSt9_Any_dataOS2_OS6_","s+":"296"},{"a":"558F8A55D967","b":"558F88CDF000","o":"187E967","s":"_ZZN5mongo30Confi gServerCatalogCacheLoader14getChunksSinceERKNS_15NamespaceStringENS_12ChunkVersionESt8functionIFvPNS_16OperationContextENS_10StatusWithINS_18CatalogCach eLoader26CollectionAndChangedChunksEEEEEENKUlT_E_clINS_6StatusEEEDaSE_","s+":"167"},{"a":"558F8A55DD77","b":"558F88CDF000","o":"187ED77","s":"_ZZN5mongo 15unique_functionIFvNS_6StatusEEE8makeImplIZNS_30ConfigServerCatalogCacheLoader14getChunksSinceERKNS_15NamespaceStringENS_12ChunkVersionESt8functionIFvP NS_16OperationContextENS_10StatusWithINS_18CatalogCacheLoader26CollectionAndChangedChunksEEEEEEUlT_E_EEDaOSJ_EN12SpecificImpl4callEOS1_","s+":"37"},{"a":"558F8B5D4832","b":"558F88CDF000","o":"28F5832","s":"_ZN5mongo10ThreadPool10_doOneTaskEPSt11unique_lockINS_12latch_detail5LatchEE","s+":"132"},{"a":"558F8B5D6E76","b":"558F88CDF000","o":"28F7E76","s":"_ZN5mongo10ThreadPool13_consumeTasksEv","s+":"86"},{"a":"558F8B5D7C21","b":"558F88CDF000","o":"28F8C21","s":"_ZN5mongo10ThreadPool17_workerThreadBodyEPS0_RKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE","s+":"E1"},{"a":"558F8B5D7F50","b":"558F88CDF000","o":"28F8F50","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_10ThreadPool25_startWorkerThread_inlockEvEUlvE2_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"60"},{"a":"558F8BC52D2F","b":"558F88CDF000","o":"2F73D2F","s":"execute_native_thread_routine","s+":"F"},{"a":"7FB729094EA5","b":"7FB72908D000","o":"7EA5","s":"start_thread","s+":"C5"},{"a":"7FB728DBD9FD","b":"7FB728CBF000","o":"FE9FD","s":"clone","s+":"6D"}],"processInfo":{"mongodbVersion":"4.4.9","gitVersion":"b4048e19814bfebac717cf5a880076aa69aba481","compiledModules":[],"uname":{"sysname":"Linux","release":"3.10.0-1160.24.1.el7.x86_64","version":"#1 SMP Thu Apr 8 19:51:47 UTC 2021","machine":"x86_64"},"somap":[{"b":"558F88CDF000","elfType":3,"buildId":"375E0455B64A8CCBA2B20814F34100164730166F"},{"b":"7FB72908D000","path":"/lib64/libpthread.so.0","elfType":3,"buildId":"E10CC8F2B932FC3DAEDA22F8DAC5EBB969524E5B"},{"b":"7FB728CBF000","path":"/lib64/libc.so.6","elfType":3,"buildId":"A317B42B15368ADCAE21C11107691A03EC91059D"}]}}}} {"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8BAA83AA","b":"558F88CDF000","o":"2DC93AA","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1EA"}}} {"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8BAA9E39","b":"558F88CDF000","o":"2DCAE39","s":"_ZN5mongo15printStackTraceEv","s+":"29"}}} {"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8BAA71C6","b":"558F88CDF000","o":"2DC81C6","s":"_ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo_tPv","s+":"66"}}} {"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"7FB72909C630","b":"7FB72908D000","o":"F630","s":"_L_unlock_13","s+":"34"}}} {"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"7FB728CF5387","b":"7FB728CBF000","o":"36387","s":"gsignal","s+":"37"}}} {"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"7FB728CF6A78","b":"7FB728CBF000","o":"37A78","s":"abort","s+":"148"}}} {"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F89C1D089","b":"558F88CDF000","o":"F3E089","s":"_ZN5mongo15invariantFailedEPKcS1_j","s+":"12C"}}} {"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F89AC7DB2","b":"558F88CDF000","o":"DE8DB2","s":"_ZN5mongo19RoutingTableHistory11makeUpdatedERKSt6vectorINS_9ChunkTypeESaIS2_EE.cold.1193","s+":"130"}}} {"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8AA4BD7B","b":"558F88CDF000","o":"1D6CD7B","s":"_ZN5mongo19RoutingTableHistory7makeNewENS_15NamespaceStringEN5boost8optionalINS_4UUIDEEENS_10KeyPatternESt10unique_ptrINS_17CollatorInterfaceESt14default_deleteIS8_EEbNS_3OIDERKSt6vectorINS_9ChunkTypeESaISE_EE","s+":"18B"}}} {"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8A578136","b":"558F88CDF000","o":"1899136","s":"_ZNSt17_Function_handlerIFvPN5mongo16OperationContextENS0_10StatusWithINS0_18CatalogCacheLoader26CollectionAndChangedChunksEEEEZNS0_12CatalogCache26_scheduleCollectionRefreshENS0_8WithLockESt10shared_ptrINS8_26CollectionRoutingInfoEntryEERKNS0_15NamespaceStringEiEUlS2_S6_E1_E9_M_invokeERKSt9_Any_dataOS2_OS6_","s+":"296"}}} {"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8A55D967","b":"558F88CDF000","o":"187E967","s":"_ZZN5mongo30ConfigServerCatalogCacheLoader14getChunksSinceERKNS_15NamespaceStringENS_12ChunkVersionESt8functionIFvPNS_16OperationContextENS_10StatusWithINS_18CatalogCacheLoader26CollectionAndChangedChunksEEEEEENKUlT_E_clINS_6StatusEEEDaSE_","s+":"167"}}} {"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8A55DD77","b":"558F88CDF000","o":"187ED77","s":"_ZZN5mongo15unique_functionIFvNS_6StatusEEE8makeImplIZNS_30ConfigServerCatalogCacheLoader14getChunksSinceERKNS_15NamespaceStringENS_12ChunkVersionESt8functionIFvPNS_16OperationContextENS_10StatusWithINS_18CatalogCacheLoader26CollectionAndChangedChunksEEEEEEUlT_E_EEDaOSJ_EN12SpecificImpl4callEOS1_","s+":"37"}}} {"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8B5D4832","b":"558F88CDF000","o":"28F5832","s":"_ZN5mongo10ThreadPool10_doOneTaskEPSt11unique_lockINS_12latch_detail5LatchEE","s+":"132"}}} {"t":{"$date":"2021-11-06T14:55:02.067+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8B5D6E76","b":"558F88CDF000","o":"28F7E76","s":"_ZN5mongo10ThreadPool13_consumeTasksEv","s+":"86"}}} {"t":{"$date":"2021-11-06T14:55:02.068+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8B5D7C21","b":"558F88CDF000","o":"28F8C21","s":"_ZN5mongo10ThreadPool17_workerThreadBodyEPS0_RKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE","s+":"E1"}}} {"t":{"$date":"2021-11-06T14:55:02.068+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8B5D7F50","b":"558F88CDF000","o":"28F8F50","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_10ThreadPool25_startWorkerThread_inlockEvEUlvE2_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"60"}}} {"t":{"$date":"2021-11-06T14:55:02.068+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"558F8BC52D2F","b":"558F88CDF000","o":"2F73D2F","s":"execute_native_thread_routine","s+":"F"}}} {"t":{"$date":"2021-11-06T14:55:02.068+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"7FB729094EA5","b":"7FB72908D000","o":"7EA5","s":"start_thread","s+":"C5"}}} {"t":{"$date":"2021-11-06T14:55:02.068+03:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"ConfigServerCatalogCacheLoader-1","msg":" Frame: {frame}","attr":{"frame":{"a":"7FB728DBD9FD","b":"7FB728CBF000","o":"FE9FD","s":"clone","s+":"6D"}}}
Downgrading does not solve the problem.
What can be done?