Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-44983

mongodb start failed

    • Type: Icon: Question Question
    • Resolution: Done
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: 3.4.10
    • Component/s: None
    • Labels:
      None
    • Environment:
      centos7

      [root@localhost mongo]# mongod --dbpath=/var/lib/mongo --repair --storageEngine wiredTiger
      2019-12-06T20:49:50.516+0800 I CONTROL  [initandlisten] MongoDB starting : pid=2600 port=27017 dbpath=/var/lib/mongo 64-bit host=localhost.localdomain
      2019-12-06T20:49:50.516+0800 I CONTROL  [initandlisten] db version v3.4.10
      2019-12-06T20:49:50.516+0800 I CONTROL  [initandlisten] git version: 078f28920cb24de0dd479b5ea6c66c644f6326e9
      2019-12-06T20:49:50.516+0800 I CONTROL  [initandlisten] OpenSSL version: OpenSSL 1.0.1e-fips 11 Feb 2013
      2019-12-06T20:49:50.516+0800 I CONTROL  [initandlisten] allocator: tcmalloc
      2019-12-06T20:49:50.516+0800 I CONTROL  [initandlisten] modules: none
      2019-12-06T20:49:50.516+0800 I CONTROL  [initandlisten] build environment:
      2019-12-06T20:49:50.516+0800 I CONTROL  [initandlisten]     distmod: rhel70
      2019-12-06T20:49:50.516+0800 I CONTROL  [initandlisten]     distarch: x86_64
      2019-12-06T20:49:50.516+0800 I CONTROL  [initandlisten]     target_arch: x86_64
      2019-12-06T20:49:50.516+0800 I CONTROL  [initandlisten] options: { repair: true, storage: { dbPath: "/var/lib/mongo", engine: "wiredTiger" } }
      2019-12-06T20:49:50.572+0800 I STORAGE  [initandlisten]
      2019-12-06T20:49:50.573+0800 I STORAGE  [initandlisten] ** WARNING: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine
      2019-12-06T20:49:50.573+0800 I STORAGE  [initandlisten] **          See http://dochub.mongodb.org/core/prodnotes-filesystem
      2019-12-06T20:49:50.573+0800 I STORAGE  [initandlisten] Detected WT journal files.  Running recovery from last checkpoint.
      2019-12-06T20:49:50.573+0800 I STORAGE  [initandlisten] journal to nojournal transition config: create,cache_size=7424M,session_max=20000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),
      2019-12-06T20:49:50.598+0800 E STORAGE  [initandlisten] WiredTiger error (0) [1575636590:598409][2600:0x7fd34612ce40], file:WiredTiger.wt, connection: read checksum error for 4096B block at offset 36864: block header checksum of 1818195054 doesn't match expected checksum of 2021188364
      2019-12-06T20:49:50.598+0800 E STORAGE  [initandlisten] WiredTiger error (0) [1575636590:598516][2600:0x7fd34612ce40], file:WiredTiger.wt, connection: WiredTiger.wt: encountered an illegal file format or internal value
      2019-12-06T20:49:50.598+0800 E STORAGE  [initandlisten] WiredTiger error (-31804) [1575636590:598538][2600:0x7fd34612ce40], file:WiredTiger.wt, connection: the process must exit and restart: WT_PANIC: WiredTiger library panic
      2019-12-06T20:49:50.598+0800 I -        [initandlisten] Fatal Assertion 28558 at src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp 361
      2019-12-06T20:49:50.598+0800 I -        [initandlisten]
      
      ***aborting after fassert() failure
      
      
      2019-12-06T20:49:50.646+0800 F -        [initandlisten] Got signal: 6 (Aborted).
      
       0x7fd3476bf6b1 0x7fd3476be8c9 0x7fd3476bedad 0x7fd344d1d5f0 0x7fd344976337 0x7fd344977a28 0x7fd346963267 0x7fd3473cfcb6 0x7fd34696d576 0x7fd34696d792 0x7fd34696d9f4 0x7fd347fc6a0f 0x7fd347fc6f5b 0x7fd347fc354d 0x7fd347fc8127 0x7fd347fe1d63 0x7fd34801b43b 0x7fd3480a627b 0x7fd3480a67bd 0x7fd3480a6a7c 0x7fd348029b61 0x7fd34809c5a8 0x7fd34806656e 0x7fd34806664b 0x7fd348017629 0x7fd3473b4d82 0x7fd3473ad215 0x7fd34729f737 0x7fd34694f36c 0x7fd34696edab 0x7fd344962505 0x7fd3469cdf9f
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"7FD346147000","o":"15786B1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"7FD346147000","o":"15778C9"},{"b":"7FD346147000","o":"1577DAD"},{"b":"7FD344D0E000","o":"F5F0"},{"b":"7FD344940000","o":"36337","s":"gsignal"},{"b":"7FD344940000","o":"37A28","s":"abort"},{"b":"7FD346147000","o":"81C267","s":"_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj"},{"b":"7FD346147000","o":"1288CB6"},{"b":"7FD346147000","o":"826576","s":"__wt_eventv"},{"b":"7FD346147000","o":"826792","s":"__wt_err"},{"b":"7FD346147000","o":"8269F4","s":"__wt_panic"},{"b":"7FD346147000","o":"1E7FA0F","s":"__wt_block_extlist_read"},{"b":"7FD346147000","o":"1E7FF5B","s":"__wt_block_extlist_read_avail"},{"b":"7FD346147000","o":"1E7C54D","s":"__wt_block_checkpoint_load"},{"b":"7FD346147000","o":"1E81127"},{"b":"7FD346147000","o":"1E9AD63","s":"__wt_btree_open"},{"b":"7FD346147000","o":"1ED443B","s":"__wt_conn_btree_open"},{"b":"7FD346147000","o":"1F5F27B","s":"__wt_session_get_btree"},{"b":"7FD346147000","o":"1F5F7BD","s":"__wt_session_get_btree"},{"b":"7FD346147000","o":"1F5FA7C","s":"__wt_session_get_btree_ckpt"},{"b":"7FD346147000","o":"1EE2B61","s":"__wt_curfile_open"},{"b":"7FD346147000","o":"1F555A8"},{"b":"7FD346147000","o":"1F1F56E","s":"__wt_metadata_cursor_open"},{"b":"7FD346147000","o":"1F1F64B","s":"__wt_metadata_cursor"},{"b":"7FD346147000","o":"1ED0629","s":"wiredtiger_open"},{"b":"7FD346147000","o":"126DD82","s":"_ZN5mongo18WiredTigerKVEngineC2ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES8_PNS_11ClockSourceES8_mbbbb"},{"b":"7FD346147000","o":"1266215"},{"b":"7FD346147000","o":"1158737","s":"_ZN5mongo20ServiceContextMongoD29initializeGlobalStorageEngineEv"},{"b":"7FD346147000","o":"80836C"},{"b":"7FD346147000","o":"827DAB","s":"main"},{"b":"7FD344940000","o":"22505","s":"__libc_start_main"},{"b":"7FD346147000","o":"886F9F"}],"processInfo":{ "mongodbVersion" : "3.4.10", "gitVersion" : "078f28920cb24de0dd479b5ea6c66c644f6326e9", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.10.0-327.el7.x86_64", "version" : "#1 SMP Thu Nov 19 22:10:57 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "b" : "7FD346147000", "elfType" : 3, "buildId" : "94C7FAB092E567C9338D13DB9B68751363D15EFD" }, { "b" : "7FFD31CFB000", "elfType" : 3, "buildId" : "0F488C85309E4AB293D3B758748A3037B01F9885" }, { "b" : "7FD345CB1000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "3B305C3BA17FE394862E749763F2956C9C890C2E" }, { "b" : "7FD34584E000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "4CF1939F660008CFA869D8364651F31AACD2C1C4" }, { "b" : "7FD345646000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "4749697BF078337576C4629F0D30B296A0939779" }, { "b" : "7FD345442000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "18113E6E83D8E981B8E8D808F7F3DBB23F950A1D" }, { "b" : "7FD345140000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "5681C054FDABCF789F4DDA66E94F1F6ED1747327" }, { "b" : "7FD344F2A000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "DAC0179F4555AEFEC9E97476201802FD20C03EC5" }, { "b" : "7FD344D0E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "8B33F7F8C86F8D544C63C5541A8E42B3DDFEF8B1" }, { "b" : "7FD344940000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "398944D32CF16A67AF51067A326E6C0CC14F90ED" }, { "b" : "7FD345F23000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "5CC1A53B747A7E4D21198723C2B633E54F3C06D9" }, { "b" : "7FD3446F4000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "D46A230FFF4A7B808B3CFC213D31FCAC542FB504" }, { "b" : "7FD34440F000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "6D6136A0E795420B05854DEF13A10C226FE9CCB2" }, { "b" : "7FD34420B000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "3A1166709F88740C49E060731832E3FAD2DFB66B" }, { "b" : "7FD343FD9000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "AA97A848DD7C9E57B06EC913E10D420AEBBCE027" }, { "b" : "7FD343DC3000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "1982C8CDAE90F898D1AD26DC07E807333B4789D0" }, { "b" : "7FD343BB4000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "AEF6C3D3C5152F339942041519A106FC055DAF71" }, { "b" : "7FD3439B0000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "2E01D5AC08C1280D013AAB96B292AC58BC30A263" }, { "b" : "7FD343797000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "3009B26B33156EAAF99787AA3DA0C6AE99649755" }, { "b" : "7FD343572000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "82FF6B18E1E42825CC2D060F969479AD4AF2F62C" }, { "b" : "7FD343310000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "9CA3D11F018BEEB719CDB34BE800BF1641350D0A" }, { "b" : "7FD3430EB000", "path" : "/lib64/liblzma.so.5", "elfType" : 3, "buildId" : "98131C9354279ABD39FD80D4BE5B3EC5678BD9E0" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x7fd3476bf6b1]
       mongod(+0x15778C9) [0x7fd3476be8c9]
       mongod(+0x1577DAD) [0x7fd3476bedad]
       libpthread.so.0(+0xF5F0) [0x7fd344d1d5f0]
       libc.so.6(gsignal+0x37) [0x7fd344976337]
       libc.so.6(abort+0x148) [0x7fd344977a28]
       mongod(_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj+0x0) [0x7fd346963267]
       mongod(+0x1288CB6) [0x7fd3473cfcb6]
       mongod(__wt_eventv+0x3D7) [0x7fd34696d576]
       mongod(__wt_err+0x9D) [0x7fd34696d792]
       mongod(__wt_panic+0x2E) [0x7fd34696d9f4]
       mongod(__wt_block_extlist_read+0x8F) [0x7fd347fc6a0f]
       mongod(__wt_block_extlist_read_avail+0x2B) [0x7fd347fc6f5b]
       mongod(__wt_block_checkpoint_load+0x26D) [0x7fd347fc354d]
       mongod(+0x1E81127) [0x7fd347fc8127]
       mongod(__wt_btree_open+0xB43) [0x7fd347fe1d63]
       mongod(__wt_conn_btree_open+0x19B) [0x7fd34801b43b]
       mongod(__wt_session_get_btree+0xFB) [0x7fd3480a627b]
       mongod(__wt_session_get_btree+0x63D) [0x7fd3480a67bd]
       mongod(__wt_session_get_btree_ckpt+0x14C) [0x7fd3480a6a7c]
       mongod(__wt_curfile_open+0x161) [0x7fd348029b61]
       mongod(+0x1F555A8) [0x7fd34809c5a8]
       mongod(__wt_metadata_cursor_open+0x6E) [0x7fd34806656e]
       mongod(__wt_metadata_cursor+0x4B) [0x7fd34806664b]
       mongod(wiredtiger_open+0x1659) [0x7fd348017629]
       mongod(_ZN5mongo18WiredTigerKVEngineC2ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES8_PNS_11ClockSourceES8_mbbbb+0xAE2) [0x7fd3473b4d82]
       mongod(+0x1266215) [0x7fd3473ad215]
       mongod(_ZN5mongo20ServiceContextMongoD29initializeGlobalStorageEngineEv+0x697) [0x7fd34729f737]
       mongod(+0x80836C) [0x7fd34694f36c]
       mongod(main+0x96B) [0x7fd34696edab]
       libc.so.6(__libc_start_main+0xF5) [0x7fd344962505]
       mongod(+0x886F9F) [0x7fd3469cdf9f]
      -----  END BACKTRACE  -----
      

        1. WiredTiger.wt
          736 kB
        2. WiredTiger.turtle
          1.0 kB
        3. repair_attempt.tar.gz
          34 kB

            Assignee:
            carl.champain@mongodb.com Carl Champain (Inactive)
            Reporter:
            ningpengchao@163.com lemon ning
            Votes:
            0 Vote for this issue
            Watchers:
            7 Start watching this issue

              Created:
              Updated:
              Resolved: