Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-60055

Mongo DB is not getting restarted in production

    XMLWordPrintable

    Details

    • Type: Bug
    • Status: Closed
    • Priority: Critical - P2
    • Resolution: Community Answered
    • Affects Version/s: 4.4.6
    • Fix Version/s: None
    • Component/s: None
    • Environment:
      OS :- Red Hat Linux 8
      Architecture: x86_64
      CPU op-mode(s): 32-bit, 64-bit
      Byte Order: Little Endian
      CPU(s): 48
      RAM - 256 GB
    • Backport Requested:
      v4.4

      Description

      We are having 4.4 TB of data ( occupied in file system ) and we have created indexes over collections that is occupying data space of 750GB. 
      For some reason, the server got restarted and when we tried to restart the mongo instance, it is not getting restarted. 
      As per logs suggestion, we tried to repair and we are getting below error. 
      Could someone help in getting the db up ? 

      {"t":\{"$date":"2021-09-17T17:51:39.517+00:00"},"s":"F", "c":"-", "id":23081, "ctx":"initandlisten","msg":"Invariant failure","attr":\{"expr":"buildUUID","msg":"collection: dynamediation-reporting.DataBilledindex:octlbfpvd","file":"src/mongo/db/catalog/index_catalog_impl.cpp","line":133}}
      {"t":\{"$date":"2021-09-17T17:51:39.517+00:00"},"s":"F", "c":"-", "id":23082, "ctx":"initandlisten",*"msg":"\n\n***aborting after invariant() failure\n\n"}*
      {"t":\{"$date":"2021-09-17T17:51:39.517+00:00"},"s":"F", "c":"CONTROL", "id":4757800, "ctx":"initandlisten",*"msg":"Writing fatal message","*attr":\{"message":"Got signal: 6 (Aborted).\n"}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31431, "ctx":"initandlisten","msg":"BACKTRACE: \{bt}","attr":\{"bt":{"backtrace":[{"a":"55D395F2FCCA","b":"55D39319F000","o":"2D90CCA","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1EA"},\{"a":"55D395F31749","b":"55D39319F000","o":"2D92749","s":"_ZN5mongo15printStackTraceEv","s+":"29"},\{"a":"55D395F2EAF6","b":"55D39319F000","o":"2D8FAF6","s":"_ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo_tPv","s+":"66"},\{"a":"7F6A789CFD80","b":"7F6A789BD000","o":"12D80","s":"funlockfile","s+":"50"},\{"a":"7F6A7863093F","b":"7F6A785F9000","o":"3793F","s":"gsignal","s+":"10F"},\{"a":"7F6A7861AC95","b":"7F6A785F9000","o":"21C95","s":"abort","s+":"127"},\{"a":"55D3940CDCDE","b":"55D39319F000","o":"F2ECDE","s":"_ZN5mongo22invariantFailedWithMsgEPKcRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES1_j","s+":"153"},\{"a":"55D393EDBA87","b":"55D39319F000","o":"D3CA87","s":"_ZN5mongo16IndexCatalogImpl4initEPNS_16OperationContextE.cold.1305","s+":"1B"},\{"a":"55D394A7F7F8","b":"55D39319F000","o":"18E07F8","s":"_ZN5mongo14CollectionImpl4initEPNS_16OperationContextE","s+":"328"},\{"a":"55D394A8BFE6","b":"55D39319F000","o":"18ECFE6","s":"_ZNK5mongo12DatabaseImpl4initEPNS_16OperationContextE","s+":"D6"},\{"a":"55D394A87618","b":"55D39319F000","o":"18E8618","s":"_ZN5mongo18DatabaseHolderImpl6openDbEPNS_16OperationContextENS_10StringDataEPb","s+":"308"},\{"a":"55D3945E7327","b":"55D39319F000","o":"1448327","s":"_ZN5mongo14repairDatabaseEPNS_16OperationContextEPNS_13StorageEngineERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE","s+":"247"},\{"a":"55D3945E20CE","b":"55D39319F000","o":"14430CE","s":"_ZN5mongo30repairDatabasesAndCheckVersionEPNS_16OperationContextE","s+":"34E"},\{"a":"55D394167292","b":"55D39319F000","o":"FC8292","s":"_ZN5mongo12_GLOBAL__N_114_initAndListenEPNS_14ServiceContextEi.isra.1583","s+":"762"},\{"a":"55D39416AD30","b":"55D39319F000","o":"FCBD30","s":"_ZN5mongo12_GLOBAL__N_111mongoDbMainEiPPcS2_","s+":"650"},\{"a":"55D3940DD9A9","b":"55D39319F000","o":"F3E9A9","s":"main","s+":"9"},\{"a":"7F6A7861C813","b":"7F6A785F9000","o":"23813","s":"__libc_start_main","s+":"F3"},\{"a":"55D39416635E","b":"55D39319F000","o":"FC735E","s":"_start","s+":"2E"}],"processInfo":\{"mongodbVersion":"4.4.6","gitVersion":"72e66213c2c3eab37d9358d5e78ad7f5c1d0d0d7","compiledModules":[],"uname":{"sysname":"Linux","release":"4.18.0-80.el8.x86_64","version":"#1 SMP Wed Mar 13 12:02:46 UTC 2019","machine":"x86_64"},"somap":[\{"b":"55D39319F000","elfType":3,"buildId":"264EA6EE10C77300B36F30AE15B879E4CE140A3C"},\{"b":"7F6A789BD000","path":"/lib64/libpthread.so.0","elfType":3,"buildId":"5326B8728FA01B7149DAC943100F1405533E76CE"},\{"b":"7F6A785F9000","path":"/lib64/libc.so.6","elfType":3,"buildId":"424987D36A35374FC6A1438711AFB5113A74EA81"}]}}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D395F2FCCA","b":"55D39319F000","o":"2D90CCA","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.606","s+":"1EA"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D395F31749","b":"55D39319F000","o":"2D92749","s":"_ZN5mongo15printStackTraceEv","s+":"29"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D395F2EAF6","b":"55D39319F000","o":"2D8FAF6","s":"_ZN5mongo12_GLOBAL__N_116abruptQuitActionEiP9siginfo_tPv","s+":"66"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"7F6A789CFD80","b":"7F6A789BD000","o":"12D80","s":"funlockfile","s+":"50"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"7F6A7863093F","b":"7F6A785F9000","o":"3793F","s":"gsignal","s+":"10F"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"7F6A7861AC95","b":"7F6A785F9000","o":"21C95","s":"abort","s+":"127"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D3940CDCDE","b":"55D39319F000","o":"F2ECDE","s":"_ZN5mongo22invariantFailedWithMsgEPKcRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES1_j","s+":"153"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D393EDBA87","b":"55D39319F000","o":"D3CA87","s":"_ZN5mongo16IndexCatalogImpl4initEPNS_16OperationContextE.cold.1305","s+":"1B"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D394A7F7F8","b":"55D39319F000","o":"18E07F8","s":"_ZN5mongo14CollectionImpl4initEPNS_16OperationContextE","s+":"328"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D394A8BFE6","b":"55D39319F000","o":"18ECFE6","s":"_ZNK5mongo12DatabaseImpl4initEPNS_16OperationContextE","s+":"D6"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D394A87618","b":"55D39319F000","o":"18E8618","s":"_ZN5mongo18DatabaseHolderImpl6openDbEPNS_16OperationContextENS_10StringDataEPb","s+":"308"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D3945E7327","b":"55D39319F000","o":"1448327","s":"_ZN5mongo14repairDatabaseEPNS_16OperationContextEPNS_13StorageEngineERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE","s+":"247"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D3945E20CE","b":"55D39319F000","o":"14430CE","s":"_ZN5mongo30repairDatabasesAndCheckVersionEPNS_16OperationContextE","s+":"34E"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D394167292","b":"55D39319F000","o":"FC8292","s":"_ZN5mongo12_GLOBAL__N_114_initAndListenEPNS_14ServiceContextEi.isra.1583","s+":"762"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D39416AD30","b":"55D39319F000","o":"FCBD30","s":"_ZN5mongo12_GLOBAL__N_111mongoDbMainEiPPcS2_","s+":"650"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D3940DD9A9","b":"55D39319F000","o":"F3E9A9","s":"main","s+":"9"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"7F6A7861C813","b":"7F6A785F9000","o":"23813","s":"__libc_start_main","s+":"F3"}}}
      {"t":\{"$date":"2021-09-17T17:51:39.556+00:00"},"s":"I", "c":"CONTROL", "id":31427, "ctx":"initandlisten","msg":" Frame: \{frame}","attr":\{"frame":{"a":"55D39416635E","b":"55D39319F000","o":"FC735E","s":"_start","s+":"2E"}}}
      

       

        Attachments

          Activity

            People

            Assignee:
            backlog-server-triage Backlog - Triage Team
            Reporter:
            gaurav.kumar@aitechbay.com Gaurav Kumar
            Participants:
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

              Dates

              Created:
              Updated:
              Resolved: