Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-18640

Wiredtiger does not recover from unclean shutdown

    • Type: Icon: Bug Bug
    • Resolution: Done
    • Priority: Icon: Critical - P2 Critical - P2
    • None
    • Affects Version/s: 3.0.3
    • Component/s: WiredTiger
    • Labels:
      None
    • ALL

      My server was shut down uncleanly. Now the database does not start

      2015-05-23T17:41:49.538+0000 I CONTROL  ***** SERVER RESTARTED *****
      2015-05-23T17:41:49.596+0000 W -        [initandlisten] Detected unclean shutdown - /mongodb_data/mongod.lock is not empty.
      2015-05-23T17:41:49.596+0000 W STORAGE  [initandlisten] Recovering data from the last clean checkpoint.
      2015-05-23T17:41:49.596+0000 I STORAGE  [initandlisten] wiredtiger_open config: create,cache_size=1G,session_max=20000,eviction=(threads_max=4),statistics=(fast),log=(enabled=true,archive=tr
      ue,path=journal,compressor=snappy),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),
      2015-05-23T17:41:49.858+0000 E STORAGE  [initandlisten] WiredTiger (2) [1432402909:858118][1876:0x7f257de42c20], file:collection-2--5585092568808516308.wt, session.open_cursor: /mongodb_data
      /collection-2--5585092568808516308.wt: No such file or directory
      2015-05-23T17:41:49.858+0000 E STORAGE  [initandlisten] no cursor for uri: table:collection-2--5585092568808516308
      2015-05-23T17:41:49.858+0000 I -        [initandlisten] Invariant failure c src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp 1032
      2015-05-23T17:41:49.871+0000 I CONTROL  [initandlisten]
       0xf6a889 0xf08321 0xeec092 0xd76f89 0xd770ac 0xd779e3 0xd77ce9 0xd71b88 0xcf1d36 0xcf4e0c 0xd70aa6 0xa8104d 0x7f3e62 0x7f93c4 0x3f3201ed5d 0x7f1bbd
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"B6A889"},{"b":"400000","o":"B08321"},{"b":"400000","o":"AEC092"},{"b":"400000","o":"976F89"},{"b":"400000","o":"9770AC"},{"b":"400000","o":"9779E3"},{"b":"40
      0000","o":"977CE9"},{"b":"400000","o":"971B88"},{"b":"400000","o":"8F1D36"},{"b":"400000","o":"8F4E0C"},{"b":"400000","o":"970AA6"},{"b":"400000","o":"68104D"},{"b":"400000","o":"3F3E62"},{"
      b":"400000","o":"3F93C4"},{"b":"3F32000000","o":"1ED5D"},{"b":"400000","o":"3F1BBD"}],"processInfo":{ "mongodbVersion" : "3.0.3", "gitVersion" : "b40106b36eecd1b4407eb1ad1af6bc60593c6105", "
      uname" : { "sysname" : "Linux", "release" : "2.6.32-431.29.2.el6.x86_64", "version" : "#1 SMP Tue Sep 9 21:36:05 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000
      ", "buildId" : "108A63CA14A4BD5E599BAC10885DBD3A85DA5439" }, { "b" : "7FFFFC6B8000", "elfType" : 3, "buildId" : "5474F0D8DAF3D6177E2C4B06F3892745CB43B4D5" }, { "path" : "/lib64/libpthread.so
      .0", "elfType" : 3, "buildId" : "B8DFF8E53D9F2B80C3C382E83EC17C828B536A39" }, { "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "934508308DAF0D5C61E9997463F0D8B0A3F096BA" }, {
       "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "A4329A30669C783FA8DEEB7D1EA83749A8FA14E1" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "583411D8786F86A1D
      6B8741C502831E6122445A7" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "454F8FC6CC6502C6401E5F9E221564D80665D277" }, { "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "b
      uildId" : "ED99110E629209C5CA6C0ED704F2C5CE3171513A" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "C9A87F6A29ED1D3CB18F539845A45FE3A9877FF1" }, { "path" : "/lib64/libgcc_s.so
      .1", "elfType" : 3, "buildId" : "A44499D29B114A5366CD72DD4883958495AC1C1D" }, { "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "6EFE254F4564519BBB80889534FAC3D61C18C387" }, { "path"
       : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "6F8E59B70E469F3A924A268911FF8FD0C37E7460" }, { "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "54BA6B78A92203
      44E77463947215E42F0EABCC62" }, { "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "6797403AA5F8FAD8ADFF683478B45F528CE4FB0E" }, { "path" : "/lib64/libcom_err.so.2", "elfType" : 3,
      "buildId" : "8CE28F280150E62296240E70ECAC64E4A57AB826" }, { "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "05733977F4E41652B86070B27A0CFC2C1EA7719D" }, { "path" : "/lib64/li
      bz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "E3FA235F3BA3F776A01A18ECA737C9890F4459
      23" }, { "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "58B69647
      8044E028A5970D48A4ED50E164B43B36" }, { "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "E6798A06BEE17CF102BBA44FD512FF8B805CEAF1" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf6a889]
       mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf08321]
       mongod(_ZN5mongo15invariantFailedEPKcS1_j+0xB2) [0xeec092]
       mongod(_ZN5mongo21WiredTigerRecordStore8Iterator7_locateERKNS_8RecordIdEb+0x159) [0xd76f89]
       mongod(_ZN5mongo21WiredTigerRecordStore8IteratorC2ERKS0_PNS_16OperationContextERKNS_8RecordIdERKNS_20CollectionScanParams9DirectionEb+0x9C) [0xd770ac]
       mongod(_ZNK5mongo21WiredTigerRecordStore11getIteratorEPNS_16OperationContextERKNS_8RecordIdERKNS_20CollectionScanParams9DirectionE+0x43) [0xd779e3]
       mongod(_ZN5mongo21WiredTigerRecordStoreC1EPNS_16OperationContextERKNS_10StringDataES5_bllPNS_28CappedDocumentDeleteCallbackEPNS_20WiredTigerSizeStorerE+0x2B9) [0xd77ce9]
       mongod(_ZN5mongo18WiredTigerKVEngine14getRecordStoreEPNS_16OperationContextERKNS_10StringDataES5_RKNS_17CollectionOptionsE+0xA8) [0xd71b88]
       mongod(_ZN5mongo22KVDatabaseCatalogEntry14initCollectionEPNS_16OperationContextERKSsb+0x276) [0xcf1d36]
       mongod(_ZN5mongo15KVStorageEngineC1EPNS_8KVEngineERKNS_22KVStorageEngineOptionsE+0x69C) [0xcf4e0c]
       mongod(+0x970AA6) [0xd70aa6]
       mongod(_ZN5mongo23GlobalEnvironmentMongoD22setGlobalStorageEngineERKSs+0x30D) [0xa8104d]
       mongod(_ZN5mongo13initAndListenEi+0x422) [0x7f3e62]
       mongod(main+0x134) [0x7f93c4]
       libc.so.6(__libc_start_main+0xFD) [0x3f3201ed5d]
       mongod(+0x3F1BBD) [0x7f1bbd]
      -----  END BACKTRACE  -----
      2015-05-23T17:41:49.871+0000 I -        [initandlisten]
      
      ***aborting after invariant() failure
      

            Assignee:
            ramon.fernandez@mongodb.com Ramon Fernandez Marina
            Reporter:
            dharshanr@scalegrid.net Dharshan Rangegowda
            Votes:
            0 Vote for this issue
            Watchers:
            8 Start watching this issue

              Created:
              Updated:
              Resolved: