Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-37521

Mongo should stop gracefully on "Too many open files"

    • Type: Icon: Bug Bug
    • Resolution: Duplicate
    • Priority: Icon: Minor - P4 Minor - P4
    • None
    • Affects Version/s: 3.6.2
    • Component/s: None
    • Labels:
      None
    • ALL

      2018-10-09T07:08:08.076+0200 F - [initandlisten] Invariant failure: ret resulted in status UnknownError: 24: Too many open files at src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp 97
      2018-10-09T07:08:08.076+0200 F - [initandlisten]

      ***aborting after invariant() failure

      2018-10-09T07:08:08.093+0200 F - [initandlisten] Got signal: 6 (Aborted).

      0x5564b3b704f1 0x5564b3b6f709 0x5564b3b6fbed 0x7f7540602390 0x7f754025c428 0x7f754025e02a 0x5564b230694f 0x5564b23cbdf6 0x5564b23cb58c 0x5564b23c09c8 0x5564b23c0a48 0x5564b23c6a61 0x5564b23b0a9c 0x5564b24fe9ee 0x5564b24f8886 0x5564b239ba30 0x5564b256bc60 0x5564b229fa10 0x5564b237a9db 0x5564b2308759 0x7f7540247830 0x5564b236a419
      ----- BEGIN BACKTRACE -----

      {"backtrace":[\{"b":"5564B197D000","o":"21F34F1","s":"_ZN5mongo15printStackTraceERSo"}

      ,{"b":"5564B197D000","o":"21F2709"},{"b":"5564B197D000","o":"21F2BED"},{"b":"7F75405F1000","o":"11390"},{"b":"7F7540227000","o":"35428","s":"gsignal"},{"b":"7F7540227000","o":"3702A","s":"abort"},{"b":"5564B197D000","o":"98994F","s":"ZN5mongo25fassertFailedWithLocationEiPKcj"},{"b":"5564B197D000","o":"A4EDF6","s":"_ZN5mongo17WiredTigerSession9getCursorERKNSt7cxx1112basic_stringIcSt11char_traitsIcESaIcEEEmb"},{"b":"5564B197D000","o":"A4E58C","s":"_ZN5mongo16WiredTigerCursorC1ERKNSt7cxx1112basic_stringIcSt11char_traitsIcESaIcEEEmbPNS_16OperationContextE"},{"b":"5564B197D000","o":"A439C8","s":"_ZN5mongo31WiredTigerRecordStoreCursorBaseC2EPNS_16OperationContextERKNS_21WiredTigerRecordStoreEb"},{"b":"5564B197D000","o":"A43A48","s":"_ZNK5mongo29StandardWiredTigerRecordStore9getCursorEPNS_16OperationContextEb"},{"b":"5564B197D000","o":"A49A61","s":"_ZN5mongo21WiredTigerRecordStore19postConstructorInitEPNS_16OperationContextE"},{"b":"5564B197D000","o":"A33A9C","s":"_ZN5mongo18WiredTigerKVEngine21getGroupedRecordStoreEPNS_16OperationContextENS_10StringDataES3_RKNS_17CollectionOptionsENS_8KVPrefixE"},{"b":"5564B197D000","o":"B819EE","s":"_ZN5mongo26KVDatabaseCatalogEntryBase14initCollectionEPNS_16OperationContextERKNSt7cxx1112basic_stringIcSt11char_traitsIcESaIcEEEb"},{"b":"5564B197D000","o":"B7B886","s":"_ZN5mongo15KVStorageEngineC2EPNS_8KVEngineERKNS_22KVStorageEngineOptionsESt8functionIFSt10unique_ptrINS_26KVDatabaseCatalogEntryBaseESt14default_deleteIS8_EENS_10StringDataEPS0_EE"},{"b":"5564B197D000","o":"A1EA30"},{"b":"5564B197D000","o":"BEEC60","s":"_ZN5mongo20ServiceContextMongoD29initializeGlobalStorageEngineEv"},{"b":"5564B197D000","o":"922A10"},{"b":"5564B197D000","o":"9FD9DB","s":"_ZN5mongo11mongoDbMainEiPPcS1"},{"b":"5564B197D000","o":"98B759","s":"main"},{"b":"7F7540227000","o":"20830","s":"__libc_start_main"},{"b":"5564B197D000","o":"9ED419","s":"_start"}],"processInfo":{ "mongodbVersion" : "3.6.2", "gitVersion" : "489d177dbd0f0420a8ca04d39fd78d0a2c539420", "compiledModules" : [], "uname" :

      { "sysname" : "Linux", "release" : "4.14.12-xxxx-std-ipv6-64", "version" : "#1 SMP Fri Jan 5 16:55:37 UTC 2018", "machine" : "x86_64" }

      , "somap" : [ { "b" : "5564B197D000", "elfType" : 3, "buildId" : "90F4CC751C09ABD90756CE2480F0217355B846B5" }, { "b" : "7FFEF5B9A000", "elfType" : 3, "buildId" : "939DB63C0F49C494B13282BD242497AF9C045C66" }, { "b" : "7F75417E6000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "E7196EFCC044747C45DA6705CBE6E3EB2CE5DAE2" }, { "b" : "7F754157D000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "DCF10134B91ED2139E3E8C72564668F5CDBA8522" }, { "b" : "7F7541139000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "1649272BE0CA9FA22F082DC86372B6C9959779B0" }, { "b" : "7F7540F31000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "F951C1E0765FCAE48F82CAFE35D1ADD36D6C9AF9" }, { "b" : "7F7540D2D000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "0FC788F0861846257B5F1773FBD438E95DFC1032" }, { "b" : "7F7540A24000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "FF7A33D389E756CA381A8189291A968EA5E1F4F8" }, { "b" : "7F754080E000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "68220AE2C65D65C1B6AAA12FA6765A6EC2F5F434" }, { "b" : "7F75405F1000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "27F189EF8DB8C3734C6A678E6EF3CB0B206D58B2" }, { "b" : "7F7540227000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "088A6E00A1814622219F346B41E775B8DD46C518" }, { "b" : "7F7541A01000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "9157F205547F0EB588E2AB1F2F120B74253A43EA" } ] }}
      mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x5564b3b704f1]
      mongod(+0x21F2709) [0x5564b3b6f709]
      mongod(+0x21F2BED) [0x5564b3b6fbed]
      libpthread.so.0(+0x11390) [0x7f7540602390]
      libc.so.6(gsignal+0x38) [0x7f754025c428]
      libc.so.6(abort+0x16A) [0x7f754025e02a]
      mongod(_ZN5mongo25fassertFailedWithLocationEiPKcj+0x0) [0x5564b230694f]
      mongod(ZN5mongo17WiredTigerSession9getCursorERKNSt7_cxx1112basic_stringIcSt11char_traitsIcESaIcEEEmb+0x106) [0x5564b23cbdf6]
      mongod(ZN5mongo16WiredTigerCursorC1ERKNSt7_cxx1112basic_stringIcSt11char_traitsIcESaIcEEEmbPNS_16OperationContextE+0x4C) [0x5564b23cb58c]
      mongod(_ZN5mongo31WiredTigerRecordStoreCursorBaseC2EPNS_16OperationContextERKNS_21WiredTigerRecordStoreEb+0x58) [0x5564b23c09c8]
      mongod(_ZNK5mongo29StandardWiredTigerRecordStore9getCursorEPNS_16OperationContextEb+0x58) [0x5564b23c0a48]
      mongod(_ZN5mongo21WiredTigerRecordStore19postConstructorInitEPNS_16OperationContextE+0x41) [0x5564b23c6a61]
      mongod(_ZN5mongo18WiredTigerKVEngine21getGroupedRecordStoreEPNS_16OperationContextENS_10StringDataES3_RKNS_17CollectionOptionsENS_8KVPrefixE+0x27C) [0x5564b23b0a9c]
      mongod(ZN5mongo26KVDatabaseCatalogEntryBase14initCollectionEPNS_16OperationContextERKNSt7_cxx1112basic_stringIcSt11char_traitsIcESaIcEEEb+0x54E) [0x5564b24fe9ee]
      mongod(_ZN5mongo15KVStorageEngineC2EPNS_8KVEngineERKNS_22KVStorageEngineOptionsESt8functionIFSt10unique_ptrINS_26KVDatabaseCatalogEntryBaseESt14default_deleteIS8_EENS_10StringDataEPS0_EE+0x9F6) [0x5564b24f8886]
      mongod(+0xA1EA30) [0x5564b239ba30]
      mongod(_ZN5mongo20ServiceContextMongoD29initializeGlobalStorageEngineEv+0x650) [0x5564b256bc60]
      mongod(+0x922A10) [0x5564b229fa10]
      mongod(ZN5mongo11mongoDbMainEiPPcS1+0x86B) [0x5564b237a9db]
      mongod(main+0x9) [0x5564b2308759]
      libc.so.6(__libc_start_main+0xF0) [0x7f7540247830]
      mongod(_start+0x29) [0x5564b236a419]
      ----- END BACKTRACE -----
      Aborted

      I know this is caused by "Too many open files" but I think mongo should not blow. Maybe there is a way to handle this situation in another form and let the database start. I set a bigger file limit and restarted without problem, just reporting. Thanks

            Assignee:
            nick.brewer Nick Brewer
            Reporter:
            sombra2eternity@gmail.com Marcos Fernándex
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

              Created:
              Updated:
              Resolved: