Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-60055

Mongo DB is not getting restarted in production

    • Type: Icon: Bug Bug
    • Resolution: Community Answered
    • Priority: Icon: Critical - P2 Critical - P2
    • None
    • Affects Version/s: 4.4.6
    • Component/s: None
    • Environment:
      OS :- Red Hat Linux 8
      Architecture: x86_64
      CPU op-mode(s): 32-bit, 64-bit
      Byte Order: Little Endian
      CPU(s): 48
      RAM - 256 GB
    • Server Triage
    • v4.4

      We are having 4.4 TB of data ( occupied in file system ) and we have created indexes over collections that is occupying data space of 750GB. 
      For some reason, the server got restarted and when we tried to restart the mongo instance, it is not getting restarted. 
      As per logs suggestion, we tried to repair and we are getting below error. 
      Could someone help in getting the db up ? 

      {"t":\{"$date":"2021-09-17T17:51:39.517+00:00"},"s":"F", "c":"-", "id":23081, "ctx":"initandlisten","msg":"Invariant failure","attr":\{"expr":"buildUUID","msg":"collection: dynamediation-reporting.DataBilledindex:octlbfpvd","file":"src/mongo/db/catalog/index_catalog_impl.cpp","line":133}}
      {"t":\{"$date":"2021-09-17T17:51:39.517+00:00"},"s":"F", "c":"-", "id":23082, "ctx":"initandlisten",*"msg":"\n\n***aborting after invariant() failure\n\n"}*
      {"t":\{"$date":"2021-09-17T17:51:39.517+00:00"},"s":"F", "c":"CONTROL", "id":4757800, "ctx":"initandlisten",*"msg":"Writing fatal message","*attr":\{"message":"Got signal: 6 (Aborted).\n"}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31431, "ctx":"initandlisten","msg":"BACKTRACE: \{bt}","attr":\{"bt":{"backtrace":[{"a":"55D395F2FCCA","b":"55D39319F000","o":"2D90CCA","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1EA"},\{"a":"55D395F31749","b":"55D39319F000","o":"2D92749","s":"_ZN5mongo15printStackTraceEv","s+":"29"},\{"a":"55D395F2EAF6","b":"55D39319F000","o":"2D8FAF6","s":"_ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo_tPv","s+":"66"},\{"a":"7F6A789CFD80","b":"7F6A789BD000","o":"12D80","s":"funlockfile","s+":"50"},\{"a":"7F6A7863093F","b":"7F6A785F9000","o":"3793F","s":"gsignal","s+":"10F"},\{"a":"7F6A7861AC95","b":"7F6A785F9000","o":"21C95","s":"abort","s+":"127"},\{"a":"55D3940CDCDE","b":"55D39319F000","o":"F2ECDE","s":"_ZN5mongo22invariantFailedWithMsgEPKcRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES1_j","s+":"153"},\{"a":"55D393EDBA87","b":"55D39319F000","o":"D3CA87","s":"_ZN5mongo16IndexCatalogImpl4initEPNS_16OperationContextE.cold.1305","s+":"1B"},\{"a":"55D394A7F7F8","b":"55D39319F000","o":"18E07F8","s":"_ZN5mongo14CollectionImpl4initEPNS_16OperationContextE","s+":"328"},\{"a":"55D394A8BFE6","b":"55D39319F000","o":"18ECFE6","s":"_ZNK5mongo12DatabaseImpl4initEPNS_16OperationContextE","s+":"D6"},\{"a":"55D394A87618","b":"55D39319F000","o":"18E8618","s":"_ZN5mongo18DatabaseHolderImpl6openDbEPNS_16OperationContextENS_10StringDataEPb","s+":"308"},\{"a":"55D3945E7327","b":"55D39319F000","o":"1448327","s":"_ZN5mongo14repairDatabaseEPNS_16OperationContextEPNS_13StorageEngineERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE","s+":"247"},\{"a":"55D3945E20CE","b":"55D39319F000","o":"14430CE","s":"_ZN5mongo30repairDatabasesAndCheckVersionEPNS_16OperationContextE","s+":"34E"},\{"a":"55D394167292","b":"55D39319F000","o":"FC8292","s":"_ZN5mongo12_GLOBAL__N_114_initAndListenEPNS_14ServiceContextEi.isra.1583","s+":"762"},\{"a":"55D39416AD30","b":"55D39319F000","o":"FCBD30","s":"_ZN5mongo12_GLOBAL__N_111mongoDbMainEiPPcS2_","s+":"650"},\{"a":"55D3940DD9A9","b":"55D39319F000","o":"F3E9A9","s":"main","s+":"9"},\{"a":"7F6A7861C813","b":"7F6A785F9000","o":"23813","s":"__libc_start_main","s+":"F3"},\{"a":"55D39416635E","b":"55D39319F000","o":"FC735E","s":"_start","s+":"2E"}],"processInfo":\{"mongodbVersion":"4.4.6","gitVersion":"72e66213c2c3eab37d9358d5e78ad7f5c1d0d0d7","compiledModules":[],"uname":{"sysname":"Linux","release":"4.18.0-80.el8.x86_64","version":"#1 SMP Wed Mar 13 12:02:46 UTC 2019","machine":"x86_64"},"somap":[\{"b":"55D39319F000","elfType":3,"buildId":"264EA6EE10C77300B36F30AE15B879E4CE140A3C"},\{"b":"7F6A789BD000","path":"/lib64/libpthread.so.0","elfType":3,"buildId":"5326B8728FA01B7149DAC943100F1405533E76CE"},\{"b":"7F6A785F9000","path":"/lib64/libc.so.6","elfType":3,"buildId":"424987D36A35374FC6A1438711AFB5113A74EA81"}]}}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D395F2FCCA","b":"55D39319F000","o":"2D90CCA","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1EA"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D395F31749","b":"55D39319F000","o":"2D92749","s":"_ZN5mongo15printStackTraceEv","s+":"29"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D395F2EAF6","b":"55D39319F000","o":"2D8FAF6","s":"_ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo_tPv","s+":"66"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"7F6A789CFD80","b":"7F6A789BD000","o":"12D80","s":"funlockfile","s+":"50"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"7F6A7863093F","b":"7F6A785F9000","o":"3793F","s":"gsignal","s+":"10F"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"7F6A7861AC95","b":"7F6A785F9000","o":"21C95","s":"abort","s+":"127"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D3940CDCDE","b":"55D39319F000","o":"F2ECDE","s":"_ZN5mongo22invariantFailedWithMsgEPKcRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES1_j","s+":"153"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D393EDBA87","b":"55D39319F000","o":"D3CA87","s":"_ZN5mongo16IndexCatalogImpl4initEPNS_16OperationContextE.cold.1305","s+":"1B"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D394A7F7F8","b":"55D39319F000","o":"18E07F8","s":"_ZN5mongo14CollectionImpl4initEPNS_16OperationContextE","s+":"328"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D394A8BFE6","b":"55D39319F000","o":"18ECFE6","s":"_ZNK5mongo12DatabaseImpl4initEPNS_16OperationContextE","s+":"D6"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D394A87618","b":"55D39319F000","o":"18E8618","s":"_ZN5mongo18DatabaseHolderImpl6openDbEPNS_16OperationContextENS_10StringDataEPb","s+":"308"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D3945E7327","b":"55D39319F000","o":"1448327","s":"_ZN5mongo14repairDatabaseEPNS_16OperationContextEPNS_13StorageEngineERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE","s+":"247"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D3945E20CE","b":"55D39319F000","o":"14430CE","s":"_ZN5mongo30repairDatabasesAndCheckVersionEPNS_16OperationContextE","s+":"34E"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D394167292","b":"55D39319F000","o":"FC8292","s":"_ZN5mongo12_GLOBAL__N_114_initAndListenEPNS_14ServiceContextEi.isra.1583","s+":"762"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D39416AD30","b":"55D39319F000","o":"FCBD30","s":"_ZN5mongo12_GLOBAL__N_111mongoDbMainEiPPcS2_","s+":"650"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D3940DD9A9","b":"55D39319F000","o":"F3E9A9","s":"main","s+":"9"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"7F6A7861C813","b":"7F6A785F9000","o":"23813","s":"__libc_start_main","s+":"F3"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D39416635E","b":"55D39319F000","o":"FC735E","s":"_start","s+":"2E"}}}
      

       

        1. mongorepair.png
          369 kB
          Gaurav Kumar

            Assignee:
            backlog-server-triage [HELP ONLY] Backlog - Triage Team
            Reporter:
            gaurav.kumar@aitechbay.com Gaurav Kumar
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

              Created:
              Updated:
              Resolved: