Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-60055

Mongo DB is not getting restarted in production

    XMLWordPrintable

Details

    • Bug
    • Status: Closed
    • Critical - P2
    • Resolution: Community Answered
    • 4.4.6
    • None
    • None
    • OS :- Red Hat Linux 8
      Architecture: x86_64
      CPU op-mode(s): 32-bit, 64-bit
      Byte Order: Little Endian
      CPU(s): 48
      RAM - 256 GB
    • Triage
    • v4.4

    Description

      We are having 4.4 TB of data ( occupied in file system ) and we have created indexes over collections that is occupying data space of 750GB. 
      For some reason, the server got restarted and when we tried to restart the mongo instance, it is not getting restarted. 
      As per logs suggestion, we tried to repair and we are getting below error. 
      Could someone help in getting the db up ? 

      {"t":\{"$date":"2021-09-17T17:51:39.517+00:00"},"s":"F", "c":"-", "id":23081, "ctx":"initandlisten","msg":"Invariant failure","attr":\{"expr":"buildUUID","msg":"collection: dynamediation-reporting.DataBilledindex:octlbfpvd","file":"src/mongo/db/catalog/index_catalog_impl.cpp","line":133}}
      {"t":\{"$date":"2021-09-17T17:51:39.517+00:00"},"s":"F", "c":"-", "id":23082, "ctx":"initandlisten",*"msg":"\n\n***aborting after invariant() failure\n\n"}*
      {"t":\{"$date":"2021-09-17T17:51:39.517+00:00"},"s":"F", "c":"CONTROL", "id":4757800, "ctx":"initandlisten",*"msg":"Writing fatal message","*attr":\{"message":"Got signal: 6 (Aborted).\n"}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31431, "ctx":"initandlisten","msg":"BACKTRACE: \{bt}","attr":\{"bt":{"backtrace":[{"a":"55D395F2FCCA","b":"55D39319F000","o":"2D90CCA","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1EA"},\{"a":"55D395F31749","b":"55D39319F000","o":"2D92749","s":"_ZN5mongo15printStackTraceEv","s+":"29"},\{"a":"55D395F2EAF6","b":"55D39319F000","o":"2D8FAF6","s":"_ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo_tPv","s+":"66"},\{"a":"7F6A789CFD80","b":"7F6A789BD000","o":"12D80","s":"funlockfile","s+":"50"},\{"a":"7F6A7863093F","b":"7F6A785F9000","o":"3793F","s":"gsignal","s+":"10F"},\{"a":"7F6A7861AC95","b":"7F6A785F9000","o":"21C95","s":"abort","s+":"127"},\{"a":"55D3940CDCDE","b":"55D39319F000","o":"F2ECDE","s":"_ZN5mongo22invariantFailedWithMsgEPKcRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES1_j","s+":"153"},\{"a":"55D393EDBA87","b":"55D39319F000","o":"D3CA87","s":"_ZN5mongo16IndexCatalogImpl4initEPNS_16OperationContextE.cold.1305","s+":"1B"},\{"a":"55D394A7F7F8","b":"55D39319F000","o":"18E07F8","s":"_ZN5mongo14CollectionImpl4initEPNS_16OperationContextE","s+":"328"},\{"a":"55D394A8BFE6","b":"55D39319F000","o":"18ECFE6","s":"_ZNK5mongo12DatabaseImpl4initEPNS_16OperationContextE","s+":"D6"},\{"a":"55D394A87618","b":"55D39319F000","o":"18E8618","s":"_ZN5mongo18DatabaseHolderImpl6openDbEPNS_16OperationContextENS_10StringDataEPb","s+":"308"},\{"a":"55D3945E7327","b":"55D39319F000","o":"1448327","s":"_ZN5mongo14repairDatabaseEPNS_16OperationContextEPNS_13StorageEngineERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE","s+":"247"},\{"a":"55D3945E20CE","b":"55D39319F000","o":"14430CE","s":"_ZN5mongo30repairDatabasesAndCheckVersionEPNS_16OperationContextE","s+":"34E"},\{"a":"55D394167292","b":"55D39319F000","o":"FC8292","s":"_ZN5mongo12_GLOBAL__N_114_initAndListenEPNS_14ServiceContextEi.isra.1583","s+":"762"},\{"a":"55D39416AD30","b":"55D39319F000","o":"FCBD30","s":"_ZN5mongo12_GLOBAL__N_111mongoDbMainEiPPcS2_","s+":"650"},\{"a":"55D3940DD9A9","b":"55D39319F000","o":"F3E9A9","s":"main","s+":"9"},\{"a":"7F6A7861C813","b":"7F6A785F9000","o":"23813","s":"__libc_start_main","s+":"F3"},\{"a":"55D39416635E","b":"55D39319F000","o":"FC735E","s":"_start","s+":"2E"}],"processInfo":\{"mongodbVersion":"4.4.6","gitVersion":"72e66213c2c3eab37d9358d5e78ad7f5c1d0d0d7","compiledModules":[],"uname":{"sysname":"Linux","release":"4.18.0-80.el8.x86_64","version":"#1 SMP Wed Mar 13 12:02:46 UTC 2019","machine":"x86_64"},"somap":[\{"b":"55D39319F000","elfType":3,"buildId":"264EA6EE10C77300B36F30AE15B879E4CE140A3C"},\{"b":"7F6A789BD000","path":"/lib64/libpthread.so.0","elfType":3,"buildId":"5326B8728FA01B7149DAC943100F1405533E76CE"},\{"b":"7F6A785F9000","path":"/lib64/libc.so.6","elfType":3,"buildId":"424987D36A35374FC6A1438711AFB5113A74EA81"}]}}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D395F2FCCA","b":"55D39319F000","o":"2D90CCA","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1EA"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D395F31749","b":"55D39319F000","o":"2D92749","s":"_ZN5mongo15printStackTraceEv","s+":"29"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D395F2EAF6","b":"55D39319F000","o":"2D8FAF6","s":"_ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo_tPv","s+":"66"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"7F6A789CFD80","b":"7F6A789BD000","o":"12D80","s":"funlockfile","s+":"50"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"7F6A7863093F","b":"7F6A785F9000","o":"3793F","s":"gsignal","s+":"10F"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"7F6A7861AC95","b":"7F6A785F9000","o":"21C95","s":"abort","s+":"127"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D3940CDCDE","b":"55D39319F000","o":"F2ECDE","s":"_ZN5mongo22invariantFailedWithMsgEPKcRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES1_j","s+":"153"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D393EDBA87","b":"55D39319F000","o":"D3CA87","s":"_ZN5mongo16IndexCatalogImpl4initEPNS_16OperationContextE.cold.1305","s+":"1B"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D394A7F7F8","b":"55D39319F000","o":"18E07F8","s":"_ZN5mongo14CollectionImpl4initEPNS_16OperationContextE","s+":"328"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D394A8BFE6","b":"55D39319F000","o":"18ECFE6","s":"_ZNK5mongo12DatabaseImpl4initEPNS_16OperationContextE","s+":"D6"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D394A87618","b":"55D39319F000","o":"18E8618","s":"_ZN5mongo18DatabaseHolderImpl6openDbEPNS_16OperationContextENS_10StringDataEPb","s+":"308"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D3945E7327","b":"55D39319F000","o":"1448327","s":"_ZN5mongo14repairDatabaseEPNS_16OperationContextEPNS_13StorageEngineERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE","s+":"247"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D3945E20CE","b":"55D39319F000","o":"14430CE","s":"_ZN5mongo30repairDatabasesAndCheckVersionEPNS_16OperationContextE","s+":"34E"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D394167292","b":"55D39319F000","o":"FC8292","s":"_ZN5mongo12_GLOBAL__N_114_initAndListenEPNS_14ServiceContextEi.isra.1583","s+":"762"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D39416AD30","b":"55D39319F000","o":"FCBD30","s":"_ZN5mongo12_GLOBAL__N_111mongoDbMainEiPPcS2_","s+":"650"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D3940DD9A9","b":"55D39319F000","o":"F3E9A9","s":"main","s+":"9"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"7F6A7861C813","b":"7F6A785F9000","o":"23813","s":"__libc_start_main","s+":"F3"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D39416635E","b":"55D39319F000","o":"FC735E","s":"_start","s+":"2E"}}}
      

       

      Attachments

        1. mongorepair.png
          369 kB
          Gaurav Kumar

        Activity

          People

            backlog-server-triage Backlog - Triage Team
            gaurav.kumar@aitechbay.com Gaurav Kumar
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved: