mongod start failed

XMLWordPrintableJSON

    • Type: Bug
    • Resolution: Duplicate
    • Priority: Major - P3
    • None
    • Affects Version/s: 3.0.2
    • Component/s: WiredTiger
    • None
    • Fully Compatible
    • ALL
    • None
    • 3
    • None
    • None
    • None
    • None
    • None
    • None

      I am using mongodb 3.0.2, storage engine - wiredTiger
      from yesterday morning i am facing this issue.
      I went through all the links where hazard_max has to be set but it didn't work for me. please help me to get out of this issue.

      2015-05-12T16:54:35.442+0530 I CONTROL ***** SERVER RESTARTED *****
2015-05-12T16:54:35.465+0530 I STORAGE [initandlisten] wiredtiger_open config: create,cache_size=7G,session_max=20000,eviction=(threads_max=4),statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),
2015-05-12T16:54:35.469+0530 E STORAGE [initandlisten] WiredTiger (0) [1431429875:469521][10501:0x7f1a672e3c20], file:WiredTiger.wt, connection: read checksum error [4096B @ 32768, 3884066602 != 1841814112]
2015-05-12T16:54:35.469+0530 E STORAGE [initandlisten] WiredTiger (0) [1431429875:469550][10501:0x7f1a672e3c20], file:WiredTiger.wt, connection: WiredTiger.wt: encountered an illegal file format or internal value
2015-05-12T16:54:35.469+0530 E STORAGE [initandlisten] WiredTiger (-31804) [1431429875:469563][10501:0x7f1a672e3c20], file:WiredTiger.wt, connection: the process must exit and restart: WT_PANIC: WiredTiger library panic
2015-05-12T16:54:35.469+0530 I - [initandlisten] Fatal Assertion 28558
2015-05-12T16:54:35.478+0530 I CONTROL [initandlisten] 
0xf68719 0xf06c91 0xeeaad1 0xd8761a 0x1399239 0x13993f5 0x1399894 0x12f0e52 0x130955c 0x1307358 0x13081d2 0x133014b 0x139877d 0x13669eb 0x132d937 0xd7150b 0xd6f3a8 0xa8033d 0x7f36a2 0x7f8c14 0x3bf4a1ed5d 0x7f1435
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"B68719"},{"b":"400000","o":"B06C91"},{"b":"400000","o":"AEAAD1"},{"b":"400000","o":"98761A"},{"b":"400000","o":"F99239"},{"b":"400000","o":"F993F5"},{"b":"400000","o":"F99894"},{"b":"400000","o":"EF0E52"},{"b":"400000","o":"F0955C"},{"b":"400000","o":"F07358"},{"b":"400000","o":"F081D2"},{"b":"400000","o":"F3014B"},{"b":"400000","o":"F9877D"},{"b":"400000","o":"F669EB"},{"b":"400000","o":"F2D937"},{"b":"400000","o":"97150B"},{"b":"400000","o":"96F3A8"},{"b":"400000","o":"68033D"},{"b":"400000","o":"3F36A2"},{"b":"400000","o":"3F8C14"},{"b":"3BF4A00000","o":"1ED5D"},{"b":"400000","o":"3F1435"}],"processInfo":{ "mongodbVersion" : "3.0.2", "gitVersion" : "6201872043ecbbc0a4cc169b5482dcf385fc464f", "uname" : { "sysname" : "Linux", "release" : "2.6.32-504.12.2.el6.x86_64", "version" : "#1 SMP Wed Mar 11 22:03:14 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "528324F7E7422ECA899108CA2A12B5AAAA2A5569" }, { "b" : "7FFFB0EFE000", "elfType" : 3, "buildId" : "4B8E1260CCD7C3D8CC131E84001F7220651617EC" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "B8DFF8E53D9F2B80C3C382E83EC17C828B536A39" }, { "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "934508308DAF0D5C61E9997463F0D8B0A3F096BA" }, { "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "A4329A30669C783FA8DEEB7D1EA83749A8FA14E1" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "583411D8786F86A1D6B8741C502831E6122445A7" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "454F8FC6CC6502C6401E5F9E221564D80665D277" }, { "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "F07F2E7CF4BFB393CC9BBE8CDC6463652E14DB07" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "7D8E9374F4A4EA38A7C1E763F32240EA113E4208" }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "246C3BAB0AB093AFD59D34C8CBF29E786DE4BE97" }, { "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "E4EAB3C200B7D8444FF95AB01F6466924A6A5F5F" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "6F8E59B70E469F3A924A268911FF8FD0C37E7460" }, { "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "54BA6B78A9220344E77463947215E42F0EABCC62" }, { "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "6797403AA5F8FAD8ADFF683478B45F528CE4FB0E" }, { "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "8CE28F280150E62296240E70ECAC64E4A57AB826" }, { "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "05733977F4E41652B86070B27A0CFC2C1EA7719D" }, { "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "E3FA235F3BA3F776A01A18ECA737C9890F445923" }, { "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F8B68F301C19BF06AF56B4B06E0A69F89D2C1F8D" }, { "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "E6798A06BEE17CF102BBA44FD512FF8B805CEAF1" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf68719]
mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf06c91]
mongod(_ZN5mongo13fassertFailedEi+0x61) [0xeeaad1]
mongod(+0x98761A) [0xd8761a]
mongod(__wt_eventv+0x489) [0x1399239]
mongod(__wt_err+0x95) [0x13993f5]
mongod(__wt_panic+0x24) [0x1399894]
mongod(__wt_bm_read+0x72) [0x12f0e52]
mongod(__wt_bt_read+0x1AC) [0x130955c]
mongod(__wt_btree_tree_open+0x58) [0x1307358]
mongod(__wt_btree_open+0xD02) [0x13081d2]
mongod(__wt_conn_btree_get+0x19B) [0x133014b]
mongod(__wt_session_get_btree+0x31D) [0x139877d]
mongod(__wt_metadata_open+0x2B) [0x13669eb]
mongod(wiredtiger_open+0xCD7) [0x132d937]
mongod(_ZN5mongo18WiredTigerKVEngineC1ERKSsS2_bb+0x2EB) [0xd7150b]
mongod(+0x96F3A8) [0xd6f3a8]
mongod(_ZN5mongo23GlobalEnvironmentMongoD22setGlobalStorageEngineERKSs+0x30D) [0xa8033d]
mongod(_ZN5mongo13initAndListenEi+0x422) [0x7f36a2]
mongod(main+0x134) [0x7f8c14]
libc.so.6(__libc_start_main+0xFD) [0x3bf4a1ed5d]
mongod(+0x3F1435) [0x7f1435]
----- END BACKTRACE -----
2015-05-12T16:54:35.478+0530 I - [initandlisten]

***aborting after fassert() failure

      when i running command for increasing hazard then the problem remain
      same.

      [root@host name ~]# mongod --dbpath /home/mongodbProject/database/wiredTiger/ --storageEngine wiredTiger --wiredTigerEngineConfigString="hazard_max=20000"

      2015-05-12T17:14:47.103+0530 I STORAGE Engine custom option: hazard_max=20000
2015-05-12T17:14:47.126+0530 I STORAGE [initandlisten] wiredtiger_open config: create,cache_size=7G,session_max=20000,eviction=(threads_max=4),statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),hazard_max=20000
2015-05-12T17:14:47.130+0530 E STORAGE [initandlisten] WiredTiger (0) [1431431087:130057][14416:0x7f83a326bc20], file:WiredTiger.wt, connection: read checksum error [4096B @ 32768, 3884066602 != 1841814112]
2015-05-12T17:14:47.130+0530 E STORAGE [initandlisten] WiredTiger (0) [1431431087:130086][14416:0x7f83a326bc20], file:WiredTiger.wt, connection: WiredTiger.wt: encountered an illegal file format or internal value
2015-05-12T17:14:47.130+0530 E STORAGE [initandlisten] WiredTiger (-31804) [1431431087:130098][14416:0x7f83a326bc20], file:WiredTiger.wt, connection: the process must exit and restart: WT_PANIC: WiredTiger library panic
2015-05-12T17:14:47.130+0530 I - [initandlisten] Fatal Assertion 28558
2015-05-12T17:14:47.139+0530 I CONTROL [initandlisten] 
0xf68719 0xf06c91 0xeeaad1 0xd8761a 0x1399239 0x13993f5 0x1399894 0x12f0e52 0x130955c 0x1307358 0x13081d2 0x133014b 0x139877d 0x13669eb 0x132d937 0xd7150b 0xd6f3a8 0xa8033d 0x7f36a2 0x7f8c14 0x3bf4a1ed5d 0x7f1435
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"B68719"},{"b":"400000","o":"B06C91"},{"b":"400000","o":"AEAAD1"},{"b":"400000","o":"98761A"},{"b":"400000","o":"F99239"},{"b":"400000","o":"F993F5"},{"b":"400000","o":"F99894"},{"b":"400000","o":"EF0E52"},{"b":"400000","o":"F0955C"},{"b":"400000","o":"F07358"},{"b":"400000","o":"F081D2"},{"b":"400000","o":"F3014B"},{"b":"400000","o":"F9877D"},{"b":"400000","o":"F669EB"},{"b":"400000","o":"F2D937"},{"b":"400000","o":"97150B"},{"b":"400000","o":"96F3A8"},{"b":"400000","o":"68033D"},{"b":"400000","o":"3F36A2"},{"b":"400000","o":"3F8C14"},{"b":"3BF4A00000","o":"1ED5D"},{"b":"400000","o":"3F1435"}],"processInfo":{ "mongodbVersion" : "3.0.2", "gitVersion" : "6201872043ecbbc0a4cc169b5482dcf385fc464f", "uname" : { "sysname" : "Linux", "release" : "2.6.32-504.12.2.el6.x86_64", "version" : "#1 SMP Wed Mar 11 22:03:14 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "528324F7E7422ECA899108CA2A12B5AAAA2A5569" }, { "b" : "7FFFE011E000", "elfType" : 3, "buildId" : "4B8E1260CCD7C3D8CC131E84001F7220651617EC" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "B8DFF8E53D9F2B80C3C382E83EC17C828B536A39" }, { "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "934508308DAF0D5C61E9997463F0D8B0A3F096BA" }, { "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "A4329A30669C783FA8DEEB7D1EA83749A8FA14E1" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "583411D8786F86A1D6B8741C502831E6122445A7" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "454F8FC6CC6502C6401E5F9E221564D80665D277" }, { "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "F07F2E7CF4BFB393CC9BBE8CDC6463652E14DB07" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "7D8E9374F4A4EA38A7C1E763F32240EA113E4208" }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "246C3BAB0AB093AFD59D34C8CBF29E786DE4BE97" }, { "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "E4EAB3C200B7D8444FF95AB01F6466924A6A5F5F" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "6F8E59B70E469F3A924A268911FF8FD0C37E7460" }, { "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "54BA6B78A9220344E77463947215E42F0EABCC62" }, { "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "6797403AA5F8FAD8ADFF683478B45F528CE4FB0E" }, { "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "8CE28F280150E62296240E70ECAC64E4A57AB826" }, { "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "05733977F4E41652B86070B27A0CFC2C1EA7719D" }, { "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "E3FA235F3BA3F776A01A18ECA737C9890F445923" }, { "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F8B68F301C19BF06AF56B4B06E0A69F89D2C1F8D" }, { "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "E6798A06BEE17CF102BBA44FD512FF8B805CEAF1" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf68719]
mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf06c91]
mongod(_ZN5mongo13fassertFailedEi+0x61) [0xeeaad1]
mongod(+0x98761A) [0xd8761a]
mongod(__wt_eventv+0x489) [0x1399239]
mongod(__wt_err+0x95) [0x13993f5]
mongod(__wt_panic+0x24) [0x1399894]
mongod(__wt_bm_read+0x72) [0x12f0e52]
mongod(__wt_bt_read+0x1AC) [0x130955c]
mongod(__wt_btree_tree_open+0x58) [0x1307358]
mongod(__wt_btree_open+0xD02) [0x13081d2]
mongod(__wt_conn_btree_get+0x19B) [0x133014b]
mongod(__wt_session_get_btree+0x31D) [0x139877d]
mongod(__wt_metadata_open+0x2B) [0x13669eb]
mongod(wiredtiger_open+0xCD7) [0x132d937]
mongod(_ZN5mongo18WiredTigerKVEngineC1ERKSsS2_bb+0x2EB) [0xd7150b]
mongod(+0x96F3A8) [0xd6f3a8]
mongod(_ZN5mongo23GlobalEnvironmentMongoD22setGlobalStorageEngineERKSs+0x30D) [0xa8033d]
mongod(_ZN5mongo13initAndListenEi+0x422) [0x7f36a2]
mongod(main+0x134) [0x7f8c14]
libc.so.6(__libc_start_main+0xFD) [0x3bf4a1ed5d]
mongod(+0x3F1435) [0x7f1435]
----- END BACKTRACE -----
2015-05-12T17:14:47.139+0530 I - [initandlisten]

***aborting after fassert() failure

        1. _mdb_catalog.wt
          48 kB
        2. SERVER-18448-fixed.zip
          44 kB
        3. SERVER-18448-fixed-2.zip
          46 kB
        4. sizeStorer.wt
          60 kB
        5. WiredTiger
          0.0 kB
        6. WiredTiger.basecfg
          0.5 kB
        7. WiredTiger.lock
          0.0 kB
        8. WiredTiger.turtle
          0.9 kB
        9. WiredTiger.wt
          708 kB

            Assignee:
            Keith Bostic (Inactive)
            Reporter:
            naveen
            Votes:
            0 Vote for this issue
            Watchers:
            10 Start watching this issue

              Created:
              Updated:
              Resolved: