Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-18448

mongod start failed

    XMLWordPrintable

    Details

    • Type: Bug
    • Status: Closed
    • Priority: Major - P3
    • Resolution: Duplicate
    • Affects Version/s: 3.0.2
    • Fix Version/s: None
    • Component/s: WiredTiger
    • Labels:
      None
    • Backwards Compatibility:
      Fully Compatible
    • Operating System:
      ALL

      Description

      I am using mongodb 3.0.2, storage engine - wiredTiger
      from yesterday morning i am facing this issue.
      I went through all the links where hazard_max has to be set but it didn't work for me. please help me to get out of this issue.

      2015-05-12T16:54:35.442+0530 I CONTROL ***** SERVER RESTARTED *****
      2015-05-12T16:54:35.465+0530 I STORAGE [initandlisten] wiredtiger_open config: create,cache_size=7G,session_max=20000,eviction=(threads_max=4),statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),
      2015-05-12T16:54:35.469+0530 E STORAGE [initandlisten] WiredTiger (0) [1431429875:469521][10501:0x7f1a672e3c20], file:WiredTiger.wt, connection: read checksum error [4096B @ 32768, 3884066602 != 1841814112]
      2015-05-12T16:54:35.469+0530 E STORAGE [initandlisten] WiredTiger (0) [1431429875:469550][10501:0x7f1a672e3c20], file:WiredTiger.wt, connection: WiredTiger.wt: encountered an illegal file format or internal value
      2015-05-12T16:54:35.469+0530 E STORAGE [initandlisten] WiredTiger (-31804) [1431429875:469563][10501:0x7f1a672e3c20], file:WiredTiger.wt, connection: the process must exit and restart: WT_PANIC: WiredTiger library panic
      2015-05-12T16:54:35.469+0530 I - [initandlisten] Fatal Assertion 28558
      2015-05-12T16:54:35.478+0530 I CONTROL [initandlisten] 
      0xf68719 0xf06c91 0xeeaad1 0xd8761a 0x1399239 0x13993f5 0x1399894 0x12f0e52 0x130955c 0x1307358 0x13081d2 0x133014b 0x139877d 0x13669eb 0x132d937 0xd7150b 0xd6f3a8 0xa8033d 0x7f36a2 0x7f8c14 0x3bf4a1ed5d 0x7f1435
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"B68719"},{"b":"400000","o":"B06C91"},{"b":"400000","o":"AEAAD1"},{"b":"400000","o":"98761A"},{"b":"400000","o":"F99239"},{"b":"400000","o":"F993F5"},{"b":"400000","o":"F99894"},{"b":"400000","o":"EF0E52"},{"b":"400000","o":"F0955C"},{"b":"400000","o":"F07358"},{"b":"400000","o":"F081D2"},{"b":"400000","o":"F3014B"},{"b":"400000","o":"F9877D"},{"b":"400000","o":"F669EB"},{"b":"400000","o":"F2D937"},{"b":"400000","o":"97150B"},{"b":"400000","o":"96F3A8"},{"b":"400000","o":"68033D"},{"b":"400000","o":"3F36A2"},{"b":"400000","o":"3F8C14"},{"b":"3BF4A00000","o":"1ED5D"},{"b":"400000","o":"3F1435"}],"processInfo":{ "mongodbVersion" : "3.0.2", "gitVersion" : "6201872043ecbbc0a4cc169b5482dcf385fc464f", "uname" : { "sysname" : "Linux", "release" : "2.6.32-504.12.2.el6.x86_64", "version" : "#1 SMP Wed Mar 11 22:03:14 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "528324F7E7422ECA899108CA2A12B5AAAA2A5569" }, { "b" : "7FFFB0EFE000", "elfType" : 3, "buildId" : "4B8E1260CCD7C3D8CC131E84001F7220651617EC" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "B8DFF8E53D9F2B80C3C382E83EC17C828B536A39" }, { "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "934508308DAF0D5C61E9997463F0D8B0A3F096BA" }, { "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "A4329A30669C783FA8DEEB7D1EA83749A8FA14E1" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "583411D8786F86A1D6B8741C502831E6122445A7" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "454F8FC6CC6502C6401E5F9E221564D80665D277" }, { "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "F07F2E7CF4BFB393CC9BBE8CDC6463652E14DB07" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "7D8E9374F4A4EA38A7C1E763F32240EA113E4208" }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "246C3BAB0AB093AFD59D34C8CBF29E786DE4BE97" }, { "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "E4EAB3C200B7D8444FF95AB01F6466924A6A5F5F" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "6F8E59B70E469F3A924A268911FF8FD0C37E7460" }, { "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "54BA6B78A9220344E77463947215E42F0EABCC62" }, { "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "6797403AA5F8FAD8ADFF683478B45F528CE4FB0E" }, { "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "8CE28F280150E62296240E70ECAC64E4A57AB826" }, { "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "05733977F4E41652B86070B27A0CFC2C1EA7719D" }, { "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "E3FA235F3BA3F776A01A18ECA737C9890F445923" }, { "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F8B68F301C19BF06AF56B4B06E0A69F89D2C1F8D" }, { "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "E6798A06BEE17CF102BBA44FD512FF8B805CEAF1" } ] }}
      mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf68719]
      mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf06c91]
      mongod(_ZN5mongo13fassertFailedEi+0x61) [0xeeaad1]
      mongod(+0x98761A) [0xd8761a]
      mongod(__wt_eventv+0x489) [0x1399239]
      mongod(__wt_err+0x95) [0x13993f5]
      mongod(__wt_panic+0x24) [0x1399894]
      mongod(__wt_bm_read+0x72) [0x12f0e52]
      mongod(__wt_bt_read+0x1AC) [0x130955c]
      mongod(__wt_btree_tree_open+0x58) [0x1307358]
      mongod(__wt_btree_open+0xD02) [0x13081d2]
      mongod(__wt_conn_btree_get+0x19B) [0x133014b]
      mongod(__wt_session_get_btree+0x31D) [0x139877d]
      mongod(__wt_metadata_open+0x2B) [0x13669eb]
      mongod(wiredtiger_open+0xCD7) [0x132d937]
      mongod(_ZN5mongo18WiredTigerKVEngineC1ERKSsS2_bb+0x2EB) [0xd7150b]
      mongod(+0x96F3A8) [0xd6f3a8]
      mongod(_ZN5mongo23GlobalEnvironmentMongoD22setGlobalStorageEngineERKSs+0x30D) [0xa8033d]
      mongod(_ZN5mongo13initAndListenEi+0x422) [0x7f36a2]
      mongod(main+0x134) [0x7f8c14]
      libc.so.6(__libc_start_main+0xFD) [0x3bf4a1ed5d]
      mongod(+0x3F1435) [0x7f1435]
      ----- END BACKTRACE -----
      2015-05-12T16:54:35.478+0530 I - [initandlisten]
       
      ***aborting after fassert() failure
      

      when i running command for increasing hazard then the problem remain
      same.

      [root@host name ~]# mongod --dbpath /home/mongodbProject/database/wiredTiger/ --storageEngine wiredTiger --wiredTigerEngineConfigString="hazard_max=20000"
      

      2015-05-12T17:14:47.103+0530 I STORAGE Engine custom option: hazard_max=20000
      2015-05-12T17:14:47.126+0530 I STORAGE [initandlisten] wiredtiger_open config: create,cache_size=7G,session_max=20000,eviction=(threads_max=4),statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),hazard_max=20000
      2015-05-12T17:14:47.130+0530 E STORAGE [initandlisten] WiredTiger (0) [1431431087:130057][14416:0x7f83a326bc20], file:WiredTiger.wt, connection: read checksum error [4096B @ 32768, 3884066602 != 1841814112]
      2015-05-12T17:14:47.130+0530 E STORAGE [initandlisten] WiredTiger (0) [1431431087:130086][14416:0x7f83a326bc20], file:WiredTiger.wt, connection: WiredTiger.wt: encountered an illegal file format or internal value
      2015-05-12T17:14:47.130+0530 E STORAGE [initandlisten] WiredTiger (-31804) [1431431087:130098][14416:0x7f83a326bc20], file:WiredTiger.wt, connection: the process must exit and restart: WT_PANIC: WiredTiger library panic
      2015-05-12T17:14:47.130+0530 I - [initandlisten] Fatal Assertion 28558
      2015-05-12T17:14:47.139+0530 I CONTROL [initandlisten] 
      0xf68719 0xf06c91 0xeeaad1 0xd8761a 0x1399239 0x13993f5 0x1399894 0x12f0e52 0x130955c 0x1307358 0x13081d2 0x133014b 0x139877d 0x13669eb 0x132d937 0xd7150b 0xd6f3a8 0xa8033d 0x7f36a2 0x7f8c14 0x3bf4a1ed5d 0x7f1435
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"B68719"},{"b":"400000","o":"B06C91"},{"b":"400000","o":"AEAAD1"},{"b":"400000","o":"98761A"},{"b":"400000","o":"F99239"},{"b":"400000","o":"F993F5"},{"b":"400000","o":"F99894"},{"b":"400000","o":"EF0E52"},{"b":"400000","o":"F0955C"},{"b":"400000","o":"F07358"},{"b":"400000","o":"F081D2"},{"b":"400000","o":"F3014B"},{"b":"400000","o":"F9877D"},{"b":"400000","o":"F669EB"},{"b":"400000","o":"F2D937"},{"b":"400000","o":"97150B"},{"b":"400000","o":"96F3A8"},{"b":"400000","o":"68033D"},{"b":"400000","o":"3F36A2"},{"b":"400000","o":"3F8C14"},{"b":"3BF4A00000","o":"1ED5D"},{"b":"400000","o":"3F1435"}],"processInfo":{ "mongodbVersion" : "3.0.2", "gitVersion" : "6201872043ecbbc0a4cc169b5482dcf385fc464f", "uname" : { "sysname" : "Linux", "release" : "2.6.32-504.12.2.el6.x86_64", "version" : "#1 SMP Wed Mar 11 22:03:14 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "528324F7E7422ECA899108CA2A12B5AAAA2A5569" }, { "b" : "7FFFE011E000", "elfType" : 3, "buildId" : "4B8E1260CCD7C3D8CC131E84001F7220651617EC" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "B8DFF8E53D9F2B80C3C382E83EC17C828B536A39" }, { "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "934508308DAF0D5C61E9997463F0D8B0A3F096BA" }, { "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "A4329A30669C783FA8DEEB7D1EA83749A8FA14E1" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "583411D8786F86A1D6B8741C502831E6122445A7" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "454F8FC6CC6502C6401E5F9E221564D80665D277" }, { "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "F07F2E7CF4BFB393CC9BBE8CDC6463652E14DB07" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "7D8E9374F4A4EA38A7C1E763F32240EA113E4208" }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "246C3BAB0AB093AFD59D34C8CBF29E786DE4BE97" }, { "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "E4EAB3C200B7D8444FF95AB01F6466924A6A5F5F" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "6F8E59B70E469F3A924A268911FF8FD0C37E7460" }, { "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "54BA6B78A9220344E77463947215E42F0EABCC62" }, { "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "6797403AA5F8FAD8ADFF683478B45F528CE4FB0E" }, { "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "8CE28F280150E62296240E70ECAC64E4A57AB826" }, { "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "05733977F4E41652B86070B27A0CFC2C1EA7719D" }, { "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "E3FA235F3BA3F776A01A18ECA737C9890F445923" }, { "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F8B68F301C19BF06AF56B4B06E0A69F89D2C1F8D" }, { "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "E6798A06BEE17CF102BBA44FD512FF8B805CEAF1" } ] }}
      mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf68719]
      mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf06c91]
      mongod(_ZN5mongo13fassertFailedEi+0x61) [0xeeaad1]
      mongod(+0x98761A) [0xd8761a]
      mongod(__wt_eventv+0x489) [0x1399239]
      mongod(__wt_err+0x95) [0x13993f5]
      mongod(__wt_panic+0x24) [0x1399894]
      mongod(__wt_bm_read+0x72) [0x12f0e52]
      mongod(__wt_bt_read+0x1AC) [0x130955c]
      mongod(__wt_btree_tree_open+0x58) [0x1307358]
      mongod(__wt_btree_open+0xD02) [0x13081d2]
      mongod(__wt_conn_btree_get+0x19B) [0x133014b]
      mongod(__wt_session_get_btree+0x31D) [0x139877d]
      mongod(__wt_metadata_open+0x2B) [0x13669eb]
      mongod(wiredtiger_open+0xCD7) [0x132d937]
      mongod(_ZN5mongo18WiredTigerKVEngineC1ERKSsS2_bb+0x2EB) [0xd7150b]
      mongod(+0x96F3A8) [0xd6f3a8]
      mongod(_ZN5mongo23GlobalEnvironmentMongoD22setGlobalStorageEngineERKSs+0x30D) [0xa8033d]
      mongod(_ZN5mongo13initAndListenEi+0x422) [0x7f36a2]
      mongod(main+0x134) [0x7f8c14]
      libc.so.6(__libc_start_main+0xFD) [0x3bf4a1ed5d]
      mongod(+0x3F1435) [0x7f1435]
      ----- END BACKTRACE -----
      2015-05-12T17:14:47.139+0530 I - [initandlisten]
       
      ***aborting after fassert() failure
      

        Attachments

        1. _mdb_catalog.wt
          48 kB
        2. SERVER-18448-fixed.zip
          44 kB
        3. SERVER-18448-fixed-2.zip
          46 kB
        4. sizeStorer.wt
          60 kB
        5. WiredTiger
          0.0 kB
        6. WiredTiger.basecfg
          0.5 kB
        7. WiredTiger.lock
          0.0 kB
        8. WiredTiger.turtle
          0.9 kB
        9. WiredTiger.wt
          708 kB

          Issue Links

            Activity

              People

              • Votes:
                0 Vote for this issue
                Watchers:
                11 Start watching this issue

                Dates

                • Created:
                  Updated:
                  Resolved: