Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-18379

DB fails to recover when specifying LSM, after system crash

    • Storage Execution
    • ALL
    • Hide
      1. Start mongod with specified options on secret port (recovery mode)
        --storageEngine wiredTiger --wiredTigerCollectionConfigString type=lsm --wiredTigerIndexConfigString type=lsm
      2. If startup is successful, kill and restart on start port
      3. Run mongo shell to randomly perform CRUD operations
      4. Crash host system at a random period (40-50 seconds after step 2)
      5. After reboot start from step 1
      Show
      Start mongod with specified options on secret port (recovery mode) --storageEngine wiredTiger --wiredTigerCollectionConfigString type=lsm --wiredTigerIndexConfigString type=lsm If startup is successful, kill and restart on start port Run mongo shell to randomly perform CRUD operations Crash host system at a random period (40-50 seconds after step 2) After reboot start from step 1

      DB recovery failed for WT when the following options were specified:
      --wiredTigerCollectionConfigString type=lsm --wiredTigerIndexConfigString type=lsm

      2015-05-07T15:57:46.891-0400 I CONTROL  ***** SERVER RESTARTED *****
      2015-05-07T15:57:46.932-0400 W -        [initandlisten] Detected unclean shutdown - /data/wiredTiger/mongod.lock is not empty.
      2015-05-07T15:57:46.932-0400 W STORAGE  [initandlisten] Recovering data from the last clean checkpoint.
      2015-05-07T15:57:46.932-0400 I STORAGE  [initandlisten] wiredtiger_open config: create,cache_size=1G,session_max=20000,eviction=(threads_max=4),statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),
      2015-05-07T15:57:49.803-0400 F -        [initandlisten] Invalid access at address: 0x18
      2015-05-07T15:57:49.932-0400 F -        [initandlisten] Got signal: 11 (Segmentation fault).
      
       0xf4ba16 0xf4b132 0xf4b46e 0x7fad6a0d3340 0x12c6cd3 0x12c2b5c 0x12c3982 0x12ea313 0x134d0d3 0x134d25d 0x12f6191 0x134b4da 0x13102b5 0x13120dc 0xd5f6a3 0xd5f85c 0xd5ff13 0xd60216 0xd5a498 0xcdbf86 0xcdffa9 0xd5940c 0xcaa860 0x8291ce 0x7ed989 0x7fad69d1fec5 0x8272f7
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"B4BA16"},{"b":"400000","o":"B4B132"},{"b":"400000","o":"B4B46E"},{"b":"7FAD6A0C3000","o":"10340"},{"b":"400000","o":"EC6CD3"},{"b":"400000","o":"EC2B5C"},{"b":"400000","o":"EC3982"},{"b":"400000","o":"EEA313"},{"b":"400000","o":"F4D0D3"},{"b":"400000","o":"F4D25D"},{"b":"400000","o":"EF6191"},{"b":"400000","o":"F4B4DA"},{"b":"400000","o":"F102B5"},{"b":"400000","o":"F120DC"},{"b":"400000","o":"95F6A3"},{"b":"400000","o":"95F85C"},{"b":"400000","o":"95FF13"},{"b":"400000","o":"960216"},{"b":"400000","o":"95A498"},{"b":"400000","o":"8DBF86"},{"b":"400000","o":"8DFFA9"},{"b":"400000","o":"95940C"},{"b":"400000","o":"8AA860"},{"b":"400000","o":"4291CE"},{"b":"400000","o":"3ED989"},{"b":"7FAD69CFE000","o":"21EC5"},{"b":"400000","o":"4272F7"}],"processInfo":{ "mongodbVersion" : "3.1.2", "gitVersion" : "aa0066050f0a9db81aa47181d0fbd18c109ae991", "uname" : { "sysname" : "Linux", "release" : "3.16.0-31-generic", "version" : "#41~14.04.1-Ubuntu SMP Wed Feb 11 19:30:13 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "5A1AB53E82C3717157B58AE3409311D6B24478D5" }, { "b" : "7FFFE552A000", "elfType" : 3, "buildId" : "5552B9335DDE9349419BA10896C1E75C9432A946" }, { "b" : "7FAD6B2E7000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "FF43D0947510134A8A494063A3C1CF3CEBB27791" }, { "b" : "7FAD6AF0D000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "379F80D2768BA6A21F52781895EE9F47B34A0A85" }, { "b" : "7FAD6AD05000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "92FCF41EFE012D6186E31A59AD05BDBB487769AB" }, { "b" : "7FAD6AB01000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "C1AE4CB7195D337A77A3C689051DABAA3980CA0C" }, { "b" : "7FAD6A7FD000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3, "buildId" : "19EFDDAB11B3BF5C71570078C59F91CF6592CE9E" }, { "b" : "7FAD6A4F7000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "1D76B71E905CB867B27CEF230FCB20F01A3178F5" }, { "b" : "7FAD6A2E1000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "8D0AA71411580EE6C08809695C3984769F25725B" }, { "b" : "7FAD6A0C3000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "9318E8AF0BFBE444731BB0461202EF57F7C39542" }, { "b" : "7FAD69CFE000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "30C94DC66A1FE95180C3D68D2B89E576D5AE213C" }, { "b" : "7FAD6B545000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "9F00581AB3C73E3AEA35995A0C50D24D59A01D47" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x26) [0xf4ba16]
       mongod(+0xB4B132) [0xf4b132]
       mongod(+0xB4B46E) [0xf4b46e]
       libpthread.so.0(+0x10340) [0x7fad6a0d3340]
       mongod(__wt_page_inmem+0x23) [0x12c6cd3]
       mongod(__wt_btree_tree_open+0x8C) [0x12c2b5c]
       mongod(__wt_btree_open+0xD02) [0x12c3982]
       mongod(__wt_conn_btree_get+0x243) [0x12ea313]
       mongod(__wt_session_get_btree+0x2C3) [0x134d0d3]
       mongod(__wt_session_get_btree_ckpt+0xBD) [0x134d25d]
       mongod(__wt_curfile_open+0xE1) [0x12f6191]
       mongod(__wt_open_cursor+0x26A) [0x134b4da]
       mongod(+0xF102B5) [0x13102b5]
       mongod(+0xF120DC) [0x13120dc]
       mongod(_ZN5mongo21WiredTigerRecordStore8Iterator7_locateERKNS_8RecordIdEb+0x173) [0xd5f6a3]
       mongod(_ZN5mongo21WiredTigerRecordStore8IteratorC2ERKS0_PNS_16OperationContextERKNS_8RecordIdERKNS_20CollectionScanParams9DirectionEb+0x9C) [0xd5f85c]
       mongod(_ZNK5mongo21WiredTigerRecordStore11getIteratorEPNS_16OperationContextERKNS_8RecordIdERKNS_20CollectionScanParams9DirectionE+0x43) [0xd5ff13]
       mongod(_ZN5mongo21WiredTigerRecordStoreC1EPNS_16OperationContextENS_10StringDataES3_bllPNS_28CappedDocumentDeleteCallbackEPNS_20WiredTigerSizeStorerE+0x2B6) [0xd60216]
       mongod(_ZN5mongo18WiredTigerKVEngine14getRecordStoreEPNS_16OperationContextENS_10StringDataES3_RKNS_17CollectionOptionsE+0x158) [0xd5a498]
       mongod(_ZN5mongo22KVDatabaseCatalogEntry14initCollectionEPNS_16OperationContextERKSsb+0x1D6) [0xcdbf86]
       mongod(_ZN5mongo15KVStorageEngineC1EPNS_8KVEngineERKNS_22KVStorageEngineOptionsE+0x579) [0xcdffa9]
       mongod(+0x95940C) [0xd5940c]
       mongod(_ZN5mongo20ServiceContextMongoD22setGlobalStorageEngineERKSs+0x350) [0xcaa860]
       mongod(_ZN5mongo13initAndListenEi+0x3EE) [0x8291ce]
       mongod(main+0x139) [0x7ed989]
       libc.so.6(__libc_start_main+0xF5) [0x7fad69d1fec5]
       mongod(+0x4272F7) [0x8272f7]
      -----  END BACKTRACE  -----
      

        1. crashserver.sh
          0.9 kB
          Jonathan Abrahams
        2. mongod-wiredTiger.log
          304 kB
          Jonathan Abrahams
        3. mongod-wiredTiger.log
          136 kB
          Jonathan Abrahams
        4. mongod-wiredTiger-recovery.log
          1.52 MB
          Jonathan Abrahams
        5. mongod-wiredTiger-recovery.log
          9 kB
          Jonathan Abrahams
        6. powertest.sh
          19 kB
          Jonathan Abrahams
        7. startmongo.sh
          2 kB
          Jonathan Abrahams
        8. wiredTiger.tar.gz
          18.30 MB
          Jonathan Abrahams
        9. wiredTiger.tar.gz
          1.72 MB
          Jonathan Abrahams
        10. wiredTiger-afterrecovery.tar.gz
          2.60 MB
          Jonathan Abrahams
        11. wiredTiger-afterrecovery.tar.gz
          7 kB
          Jonathan Abrahams
        12. wiredTiger-beforerecovery.tar.gz
          38.18 MB
          Jonathan Abrahams
        13. wiredTiger-beforerecovery.tar.gz
          3.47 MB
          Jonathan Abrahams
        14. wiredTiger-firstrun.tar.gz
          7 kB
          Jonathan Abrahams
        15. wiredTiger-firstrun.tar.gz
          7 kB
          Jonathan Abrahams

            Assignee:
            backlog-server-execution [DO NOT USE] Backlog - Storage Execution Team
            Reporter:
            jonathan.abrahams Jonathan Abrahams
            Votes:
            0 Vote for this issue
            Watchers:
            7 Start watching this issue

              Created:
              Updated:
              Resolved: