[SERVER-16210] stack trace/error on shutdown in wiredtiger Created: 18/Nov/14  Updated: 10/Dec/14  Resolved: 19/Nov/14

Status: Closed
Project: Core Server
Component/s: Storage
Affects Version/s: None
Fix Version/s: None

Type: Bug Priority: Major - P3
Reporter: Asya Kamsky Assignee: Unassigned
Resolution: Done Votes: 0
Labels: wiredtiger
Remaining Estimate: Not Specified
Time Spent: Not Specified
Original Estimate: Not Specified

Issue Links:
Duplicate
is duplicated by SERVER-16214 fatal assertion on checkpoint Closed
Operating System: ALL
Participants:

 Description   

On shutdown got this in the logs:

2014-11-17T19:30:36.147-0600 I STORAGE  [signalProcessingThread] WiredTigerKVEngine shutting down
2014-11-17T19:30:45.193-0600 E STORAGE  [signalProcessingThread] WiredTiger (0) [1416274245:193523][22388:0x7ff5ffdf2700], file:WiredTiger.wt, sweep-server: read checksum error [4096B @ 8192, 1510178857 != 2768500891]
2014-11-17T19:30:45.193-0600 E STORAGE  [signalProcessingThread] WiredTiger (0) [1416274245:193624][22388:0x7ff5ffdf2700], file:WiredTiger.wt, sweep-server: WiredTiger.wt: encountered an illegal file format or internal value
2014-11-17T19:30:45.193-0600 E STORAGE  [signalProcessingThread] WiredTiger (-31803) [1416274245:193663][22388:0x7ff5ffdf2700], file:WiredTiger.wt, sweep-server: the process must exit and restart: WT_PANIC: WiredTiger library panic
2014-11-17T19:30:45.193-0600 I -        [signalProcessingThread] Fatal Assertion 28558
2014-11-17T19:30:45.212-0600 I CONTROL  [signalProcessingThread]
 0xfd6f8e 0xf889af 0xf73050 0xdfd605 0x1445faf 0x1446326 0x14464a4 0x13a8065 0x13a7221 0x13bfec1 0x13ddc5b 0x13e2b9e 0x13cc2c9 0x144d71a 0x144d929 0x13f1518 0x13f15c7 0x13f1849 0x13f37af 0x13ebfe8 0xdfe1fa 0xaf8104 0xfd5e6b 0x101282c 0x31dac079d1 0x31da8e89dd
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"BD6F8E"},{"b":"400000","o":"B889AF"},{"b":"400000","o":"B73050"},{"b":"400000","o":"9FD605"},{"b":"400000","o":"1045FAF"},{"b":"400000","o":"1046326"},{"b":"400000","o":"10464A4"},{"b":"400000","o":"FA8065"},{"b":"400000","o":"FA7221"},{"b":"400000","o":"FBFEC1"},{"b":"400000","o":"FDDC5B"},{"b":"400000","o":"FE2B9E"},{"b":"400000","o":"FCC2C9"},{"b":"400000","o":"104D71A"},{"b":"400000","o":"104D929"},{"b":"400000","o":"FF1518"},{"b":"400000","o":"FF15C7"},{"b":"400000","o":"FF1849"},{"b":"400000","o":"FF37AF"},{"b":"400000","o":"FEBFE8"},{"b":"400000","o":"9FE1FA"},{"b":"400000","o":"6F8104"},{"b":"400000","o":"BD5E6B"},{"b":"400000","o":"C1282C"},{"b":"31DAC00000","o":"79D1"},{"b":"31DA800000","o":"E89DD"}],"processInfo":{ "mongodbVersion" : "2.8.0-rc1-pre-", "gitVersion" : "7f96d80fa8fcad7fb59bebae7e83db1a9f0120ad", "uname" : { "sysname" : "Linux", "release" : "2.6.32-504.el6.x86_64", "version" : "#1 SMP Wed Oct 15 04:27:16 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "AC5F74683E5B4906BF98D68F4F9B31D198F7A711" }, { "b" : "7FFF9F6FF000", "elfType" : 3, "buildId" : "08E42C6C3D2CD1E5D68A43B717C9EB3D310F2DF0" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "B8DFF8E53D9F2B80C3C382E83EC17C828B536A39" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "583411D8786F86A1D6B8741C502831E6122445A7" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "454F8FC6CC6502C6401E5F9E221564D80665D277" }, { "b" : "7FC42AC90000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "F07F2E7CF4BFB393CC9BBE8CDC6463652E14DB07" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "7D8E9374F4A4EA38A7C1E763F32240EA113E4208" }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "246C3BAB0AB093AFD59D34C8CBF29E786DE4BE97" }, { "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "494ED3AE74E066D8F23DD697D3D95D4F8317ADBE" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "29EA72F5B09ADC9E955977F5AEFDA978C9AE4C03" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x2E) [0xfd6f8e]
 mongod(_ZN5mongo10logContextEPKc+0xDF) [0xf889af]
 mongod(_ZN5mongo13fassertFailedEi+0xC0) [0xf73050]
 mongod(+0x9FD605) [0xdfd605]
 mongod(__wt_eventv+0x3DF) [0x1445faf]
 mongod(__wt_err+0x96) [0x1446326]
 mongod(__wt_panic+0x24) [0x14464a4]
 mongod(__wt_block_extlist_read+0x95) [0x13a8065]
 mongod(__wt_block_checkpoint+0x601) [0x13a7221]
 mongod(__wt_bt_write+0x1E1) [0x13bfec1]
 mongod(+0xFDDC5B) [0x13ddc5b]
 mongod(__wt_rec_write+0x86E) [0x13e2b9e]
 mongod(__wt_cache_op+0x139) [0x13cc2c9]
 mongod(+0x104D71A) [0x144d71a]
 mongod(__wt_checkpoint_close+0x49) [0x144d929]
 mongod(__wt_conn_btree_sync_and_close+0xF8) [0x13f1518]
 mongod(__wt_conn_dhandle_discard_single+0x87) [0x13f15c7]
 mongod(__wt_conn_dhandle_discard+0x79) [0x13f1849]
 mongod(__wt_connection_close+0xCF) [0x13f37af]
 mongod(+0xFEBFE8) [0x13ebfe8]
 mongod(_ZN5mongo18WiredTigerKVEngine13cleanShutdownEPNS_16OperationContextE+0x8A) [0xdfe1fa]
 mongod(_ZN5mongo11exitCleanlyENS_8ExitCodeE+0x2E4) [0xaf8104]
 mongod(+0xBD5E6B) [0xfd5e6b]
 mongod(+0xC1282C) [0x101282c]
 libpthread.so.0(+0x79D1) [0x31dac079d1]
 libc.so.6(clone+0x6D) [0x31da8e89dd]
-----  END BACKTRACE  -----
2014-11-17T19:30:45.212-0600 I -        [signalProcessingThread]
 
***aborting after fassert() failure
 
 
2014-11-17T19:30:45.230-0600 F -        [signalProcessingThread] Got signal: 6 (Aborted).
 
 0xfd6f8e 0xfd6a18 0xfd6d01 0x31da8326a0 0x31da832625 0x31da833e05 0xf730de 0xdfd605 0x1445faf 0x1446326 0x14464a4 0x13a8065 0x13a7221 0x13bfec1 0x13ddc5b 0x13e2b9e 0x13cc2c9 0x144d71a 0x144d929 0x13f1518 0x13f15c7 0x13f1849 0x13f37af 0x13ebfe8 0xdfe1fa 0xaf8104 0xfd5e6b 0x101282c 0x31dac079d1 0x31da8e89dd
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"BD6F8E"},{"b":"400000","o":"BD6A18"},{"b":"400000","o":"BD6D01"},{"b":"31DA800000","o":"326A0"},{"b":"31DA800000","o":"32625"},{"b":"31DA800000","o":"33E05"},{"b":"400000","o":"B730DE"},{"b":"400000","o":"9FD605"},{"b":"400000","o":"1045FAF"},{"b":"400000","o":"1046326"},{"b":"400000","o":"10464A4"},{"b":"400000","o":"FA8065"},{"b":"400000","o":"FA7221"},{"b":"400000","o":"FBFEC1"},{"b":"400000","o":"FDDC5B"},{"b":"400000","o":"FE2B9E"},{"b":"400000","o":"FCC2C9"},{"b":"400000","o":"104D71A"},{"b":"400000","o":"104D929"},{"b":"400000","o":"FF1518"},{"b":"400000","o":"FF15C7"},{"b":"400000","o":"FF1849"},{"b":"400000","o":"FF37AF"},{"b":"400000","o":"FEBFE8"},{"b":"400000","o":"9FE1FA"},{"b":"400000","o":"6F8104"},{"b":"400000","o":"BD5E6B"},{"b":"400000","o":"C1282C"},{"b":"31DAC00000","o":"79D1"},{"b":"31DA800000","o":"E89DD"}],"processInfo":{ "mongodbVersion" : "2.8.0-rc1-pre-", "gitVersion" : "7f96d80fa8fcad7fb59bebae7e83db1a9f0120ad", "uname" : { "sysname" : "Linux", "release" : "2.6.32-504.el6.x86_64", "version" : "#1 SMP Wed Oct 15 04:27:16 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "AC5F74683E5B4906BF98D68F4F9B31D198F7A711" }, { "b" : "7FFF9F6FF000", "elfType" : 3, "buildId" : "08E42C6C3D2CD1E5D68A43B717C9EB3D310F2DF0" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "B8DFF8E53D9F2B80C3C382E83EC17C828B536A39" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "583411D8786F86A1D6B8741C502831E6122445A7" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "454F8FC6CC6502C6401E5F9E221564D80665D277" }, { "b" : "7FC42AC90000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "F07F2E7CF4BFB393CC9BBE8CDC6463652E14DB07" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "7D8E9374F4A4EA38A7C1E763F32240EA113E4208" }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "246C3BAB0AB093AFD59D34C8CBF29E786DE4BE97" }, { "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "494ED3AE74E066D8F23DD697D3D95D4F8317ADBE" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "29EA72F5B09ADC9E955977F5AEFDA978C9AE4C03" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x2E) [0xfd6f8e]
 mongod(+0xBD6A18) [0xfd6a18]
 mongod(+0xBD6D01) [0xfd6d01]
 libc.so.6(+0x326A0) [0x31da8326a0]
 libc.so.6(gsignal+0x35) [0x31da832625]
 libc.so.6(abort+0x175) [0x31da833e05]
 mongod(_ZN5mongo13fassertFailedEi+0x14E) [0xf730de]
 mongod(+0x9FD605) [0xdfd605]
 mongod(__wt_eventv+0x3DF) [0x1445faf]
 mongod(__wt_err+0x96) [0x1446326]
 mongod(__wt_panic+0x24) [0x14464a4]
 mongod(__wt_block_extlist_read+0x95) [0x13a8065]
 mongod(__wt_block_checkpoint+0x601) [0x13a7221]
 mongod(__wt_bt_write+0x1E1) [0x13bfec1]
 mongod(+0xFDDC5B) [0x13ddc5b]
 mongod(__wt_rec_write+0x86E) [0x13e2b9e]
 mongod(__wt_cache_op+0x139) [0x13cc2c9]
 mongod(+0x104D71A) [0x144d71a]
 mongod(__wt_checkpoint_close+0x49) [0x144d929]
 mongod(__wt_conn_btree_sync_and_close+0xF8) [0x13f1518]
 mongod(__wt_conn_dhandle_discard_single+0x87) [0x13f15c7]
 mongod(__wt_conn_dhandle_discard+0x79) [0x13f1849]
 mongod(__wt_connection_close+0xCF) [0x13f37af]
 mongod(+0xFEBFE8) [0x13ebfe8]
 mongod(_ZN5mongo18WiredTigerKVEngine13cleanShutdownEPNS_16OperationContextE+0x8A) [0xdfe1fa]
 mongod(_ZN5mongo11exitCleanlyENS_8ExitCodeE+0x2E4) [0xaf8104]
 mongod(+0xBD5E6B) [0xfd5e6b]
 mongod(+0xC1282C) [0x101282c]
 libpthread.so.0(+0x79D1) [0x31dac079d1]
 libc.so.6(clone+0x6D) [0x31da8e89dd]
-----  END BACKTRACE  -----

Latest pull from master:

git version: 7f96d80fa8fcad7fb59bebae7e83db1a9f0120ad



 Comments   
Comment by Asya Kamsky [ 27/Nov/14 ]

You must make sure that you don't have any files in the directory that don't belong and that are updated outside of WT's "knowledge".

In my case I was shutting down mongod, deleting and recreating the directory but I was doing it without waiting for the full shutdown, so WT files were changed (by me, as it turns out).

If you are doing something like this, make sure you use something like killall -w mongod (to wait for it to shut down) before restarting or doing anything to the directory or files.

There was no "bug" here - it was interference with DB files outside the process that was causing this.

Comment by Dmitriy Selivanov [ 26/Nov/14 ]

I recieve the the same error after restarting server after many updates job (~21M). How can I start it now? Tried mongod --repair, no success.

2014-11-26T16:29:59.515+0000 I CONTROL [signalProcessingThread] got signal 15 (Terminated), will terminate after current cmd ends
2014-11-26T16:29:59.521+0000 I COMMAND [signalProcessingThread] now exiting
2014-11-26T16:29:59.521+0000 I NETWORK [signalProcessingThread] shutdown: going to close listening sockets...
2014-11-26T16:29:59.521+0000 I NETWORK [signalProcessingThread] closing listening socket: 5
2014-11-26T16:29:59.521+0000 I NETWORK [signalProcessingThread] closing listening socket: 6
2014-11-26T16:29:59.522+0000 I NETWORK [signalProcessingThread] removing socket file: /tmp/mongodb-27017.sock
2014-11-26T16:29:59.522+0000 I NETWORK [signalProcessingThread] shutdown: going to flush diaglog...
2014-11-26T16:29:59.522+0000 I NETWORK [signalProcessingThread] shutdown: going to close sockets...
2014-11-26T16:29:59.529+0000 I STORAGE [signalProcessingThread] WiredTigerKVEngine shutting down
2014-11-26T16:30:02.313+0000 I COMMAND [signalProcessingThread] dbexit: rc: 0
2014-11-26T16:30:04.115+0000 I CONTROL ***** SERVER RESTARTED *****
2014-11-26T16:30:04.120+0000 I CONTROL [initandlisten] MongoDB starting : pid=21724 port=27017 dbpath=/storage/data 64-bit host=ip-10-85-129-139
2014-11-26T16:30:04.120+0000 I CONTROL [initandlisten] db version v2.8.0-rc2-pre-
2014-11-26T16:30:04.120+0000 I CONTROL [initandlisten] git version: e4329ae5e7bfb56ef9b63a8388285b70b7ec9634
2014-11-26T16:30:04.120+0000 I CONTROL [initandlisten] build info: Linux build11.nj1.10gen.cc 2.6.32-431.3.1.el6.x86_64 #1 SMP Fri Jan 3 21:39:27 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49
2014-11-26T16:30:04.120+0000 I CONTROL [initandlisten] allocator: tcmalloc
2014-11-26T16:30:04.120+0000 I CONTROL [initandlisten] options: { config: "/etc/mongod.conf", net:

{ port: 27017 }

, storage: { dbPath: "/storage/data", engine: "wiredtiger", indexBuildRetry: false, journa
l:

{ enabled: true }

, wiredTiger:

{ engineConfig: "cache_size=26G" }

}, systemLog:

{ destination: "file", logAppend: true, path: "/storage/logs/mongodb.log" }

}
2014-11-26T16:30:04.120+0000 W STORAGE [initandlisten] Detected configuration for non-active storage engine wiredTiger when current storage engine is wiredtiger
2014-11-26T16:30:04.120+0000 I STORAGE [initandlisten] wiredtiger_open config: create,cache_size=14G,session_max=20000,extensions=[local=(entry=index_collator_extension)],statistics=(all),log=(enabled=tru
e,archive=true,path=journal),checkpoint=(wait=60,log_size=2GB),cache_size=26G
2014-11-26T16:30:04.350+0000 E STORAGE [initandlisten] WiredTiger (0) [1417019404:350401][21724:0x7f62c519cbc0], file:sizeStorer.wt, session.open_cursor: read checksum error [4096B @ 28672, 1074498338 !=
3558255388]
2014-11-26T16:30:04.350+0000 E STORAGE [initandlisten] WiredTiger (0) [1417019404:350453][21724:0x7f62c519cbc0], file:sizeStorer.wt, session.open_cursor: sizeStorer.wt: encountered an illegal file format
or internal value
2014-11-26T16:30:04.350+0000 E STORAGE [initandlisten] WiredTiger (-31803) [1417019404:350466][21724:0x7f62c519cbc0], file:sizeStorer.wt, session.open_cursor: the process must exit and restart: WT_PANIC:
WiredTiger library panic
2014-11-26T16:30:04.350+0000 I - [initandlisten] Fatal Assertion 28558
2014-11-26T16:30:04.379+0000 I CONTROL [initandlisten]
0xf41389 0xee8d61 0xece041 0xd5359a 0x1365610 0x13658d5 0x1365d41 0x12c5abe 0x12c5f08 0x12c3363 0x12c6c06 0x12dde59 0x13139fb 0x1364c38 0x1364d9a 0x131f95d 0x136397a 0x1330463 0x13637c1 0x1363be5 0xd65388
0xd54708 0xd52f30 0xa6a708 0x7da5ec 0x7df304 0x7f62c3d98ec5 0x7d8259
----- BEGIN BACKTRACE -----
{"backtrace":[

{"b":"400000","o":"B41389"}

,

{"b":"400000","o":"AE8D61"}

,

{"b":"400000","o":"ACE041"}

,

{"b":"400000","o":"95359A"}

,

{"b":"400000","o":"F65610"}

,

{"b":"400000","o":"F658D5"}

,

{"b":"400000","o":"F65D 41"}

,

{"b":"400000","o":"EC5ABE"}

,

{"b":"400000","o":"EC5F08"}

,

{"b":"400000","o":"EC3363"}

,

{"b":"400000","o":"EC6C06"}

,

{"b":"400000","o":"EDDE59"}

,

{"b":"400000","o":"F139FB"}

,

{"b":"400000","o":"F64C38"}

,

{"b" :"400000","o":"F64D9A"}

,

{"b":"400000","o":"F1F95D"}

,

{"b":"400000","o":"F6397A"}

,

{"b":"400000","o":"F30463"}

,

{"b":"400000","o":"F637C1"}

,

{"b":"400000","o":"F63BE5"}

,

{"b":"400000","o":"965388"}

,

{"b":"400000" ,"o":"954708"}

,

{"b":"400000","o":"952F30"}

,

{"b":"400000","o":"66A708"}

,

{"b":"400000","o":"3DA5EC"}

,

{"b":"400000","o":"3DF304"}

,

{"b":"7F62C3D77000","o":"21EC5"}

,

{"b":"400000","o":"3D8259"}

],"processInfo":{
"mongodbVersion" : "2.8.0-rc2-pre-", "gitVersion" : "e4329ae5e7bfb56ef9b63a8388285b70b7ec9634", "uname" :

{ "sysname" : "Linux", "release" : "3.13.0-36-generic", "version" : "#63-Ubuntu SMP Wed Sep 3 21:30 :07 UTC 2014", "machine" : "x86_64" }

, "somap" : [

{ "elfType" : 2, "b" : "400000" }

,

{ "b" : "7FFFA56E8000", "elfType" : 3 }

,

{ "b" : "7F62C4D69000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elf Type" : 3 }

,

{ "b" : "7F62C4B61000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3 }

,

{ "b" : "7F62C495D000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3 }

,

{ "b" : "7F62C465900 0", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3 }

,

{ "b" : "7F62C4353000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3 }

,

{ "b" : "7F62C413D000", "path" : "/lib/x86_64- linux-gnu/libgcc_s.so.1", "elfType" : 3 }

,

{ "b" : "7F62C3D77000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3 }

,

{ "b" : "7F62C4F87000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3 }

] }}
mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf41389]
mongod(_ZN5mongo10logContextEPKc+0xE1) [0xee8d61]
mongod(_ZN5mongo13fassertFailedEi+0x61) [0xece041]
mongod(+0x95359A) [0xd5359a]
mongod(+0xF65610) [0x1365610]
mongod(__wt_err+0x95) [0x13658d5]
mongod(__wt_panic+0x21) [0x1365d41]
mongod(__wt_block_extlist_read+0x6E) [0x12c5abe]
mongod(__wt_block_extlist_read_avail+0x28) [0x12c5f08]
mongod(__wt_block_checkpoint_load+0x1B3) [0x12c3363]
mongod(+0xEC6C06) [0x12c6c06]
mongod(__wt_btree_open+0xA19) [0x12dde59]
mongod(__wt_conn_btree_get+0x1AB) [0x13139fb]
mongod(__wt_session_get_btree+0x418) [0x1364c38]
mongod(__wt_session_get_btree_ckpt+0xCA) [0x1364d9a]
mongod(__wt_curfile_open+0x1FD) [0x131f95d]
mongod(__wt_open_cursor+0x26A) [0x136397a]
mongod(__wt_curtable_open+0x263) [0x1330463]
mongod(__wt_open_cursor+0xB1) [0x13637c1]
mongod(+0xF63BE5) [0x1363be5]
mongod(_ZN5mongo20WiredTigerSizeStorer8loadFromEPNS_17WiredTigerSessionERKSs+0x88) [0xd65388]
mongod(_ZN5mongo18WiredTigerKVEngineC2ERKSsS2_b+0x358) [0xd54708]
mongod(+0x952F30) [0xd52f30]
mongod(_ZN5mongo23GlobalEnvironmentMongoD22setGlobalStorageEngineERKSs+0x138) [0xa6a708]
mongod(_ZN5mongo13initAndListenEi+0x6EC) [0x7da5ec]
mongod(main+0x134) [0x7df304]
libc.so.6(__libc_start_main+0xF5) [0x7f62c3d98ec5]
mongod(+0x3D8259) [0x7d8259]
----- END BACKTRACE -----
2014-11-26T16:30:04.379+0000 I - [initandlisten]

***aborting after fassert() failure

2014-11-26T16:30:04.391+0000 F - [initandlisten] Got signal: 6 (Aborted).

0xf41389 0xf40cf2 0xf41076 0x7f62c3dadc30 0x7f62c3dadbb9 0x7f62c3db0fc8 0xece086 0xd5359a 0x1365610 0x13658d5 0x1365d41 0x12c5abe 0x12c5f08 0x12c3363 0x12c6c06 0x12dde59 0x13139fb 0x1364c38 0x1364d9a 0x131f95d 0x136397a 0x1330463 0x13637c1 0x1363be5 0xd65388 0xd54708 0xd52f30 0xa6a708 0x7da5ec 0x7df304 0x7f62c3d98ec5 0x7d8259
----- BEGIN BACKTRACE -----
{"backtrace":[

{"b":"400000","o":"B41389"}

,

{"b":"400000","o":"B40CF2"}

,

{"b":"400000","o":"B41076"}

,

{"b":"7F62C3D77000","o":"36C30"}

,

{"b":"7F62C3D77000","o":"36BB9"}

,

{"b":"7F62C3D77000","o":"39FC8"}

,

{"b":"400000","o":"ACE086"}

,

{"b":"400000","o":"95359A"}

,

{"b":"400000","o":"F65610"}

,

{"b":"400000","o":"F658D5"}

,

{"b":"400000","o":"F65D41"}

,

{"b":"400000","o":"EC5ABE"}

,

{"b":"400000","o":"EC5F08"}

,

{"b":"400000","o":"EC3363"}

,

{"b":"400000","o":"EC6C06"}

,

{"b":"400000","o":"EDDE59"}

,

{"b":"400000","o":"F139FB"}

,

{"b":"400000","o":"F64C38"}

,

{"b":"400000","o":"F64D9A"}

,

{"b":"400000","o":"F1F95D"}

,

{"b":"400000","o":"F6397A"}

,

{"b":"400000","o":"F30463"}

,

{"b":"400000","o":"F637C1"}

,

{"b":"400000","o":"F63BE5"}

,

{"b":"400000","o":"965388"}

,

{"b":"400000","o":"954708"}

,

{"b":"400000","o":"952F30"}

,

{"b":"400000","o":"66A708"}

,

{"b":"400000","o":"3DA5EC"}

,

{"b":"400000","o":"3DF304"}

,

{"b":"7F62C3D77000","o":"21EC5"}

,

{"b":"400000","o":"3D8259"}

],"processInfo":{ "mongodbVersion" : "2.8.0-rc2-pre-", "gitVersion" : "e4329ae5e7bfb56ef9b63a8388285b70b7ec9634", "uname" :

{ "sysname" : "Linux", "release" : "3.13.0-36-generic", "version" : "#63-Ubuntu SMP Wed Sep 3 21:30:07 UTC 2014", "machine" : "x86_64" }

, "somap" : [

{ "elfType" : 2, "b" : "400000" }

,

{ "b" : "7FFFA56E8000", "elfType" : 3 }

,

{ "b" : "7F62C4D69000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3 }

,

{ "b" : "7F62C4B61000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3 }

,

{ "b" : "7F62C495D000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3 }

,

{ "b" : "7F62C4659000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3 }

,

{ "b" : "7F62C4353000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3 }

,

{ "b" : "7F62C413D000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3 }

,

{ "b" : "7F62C3D77000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3 }

,

{ "b" : "7F62C4F87000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3 }

] }}
mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf41389]
mongod(+0xB40CF2) [0xf40cf2]
mongod(+0xB41076) [0xf41076]
libc.so.6(+0x36C30) [0x7f62c3dadc30]
libc.so.6(gsignal+0x39) [0x7f62c3dadbb9]
libc.so.6(abort+0x148) [0x7f62c3db0fc8]
mongod(_ZN5mongo13fassertFailedEi+0xA6) [0xece086]
mongod(+0x95359A) [0xd5359a]
mongod(+0xF65610) [0x1365610]
mongod(__wt_err+0x95) [0x13658d5]
mongod(__wt_panic+0x21) [0x1365d41]
mongod(__wt_block_extlist_read+0x6E) [0x12c5abe]
mongod(__wt_block_extlist_read_avail+0x28) [0x12c5f08]
mongod(__wt_block_checkpoint_load+0x1B3) [0x12c3363]
mongod(+0xEC6C06) [0x12c6c06]
mongod(__wt_btree_open+0xA19) [0x12dde59]
mongod(__wt_conn_btree_get+0x1AB) [0x13139fb]
mongod(__wt_session_get_btree+0x418) [0x1364c38]
mongod(__wt_session_get_btree_ckpt+0xCA) [0x1364d9a]
mongod(__wt_curfile_open+0x1FD) [0x131f95d]
mongod(__wt_open_cursor+0x26A) [0x136397a]
mongod(__wt_curtable_open+0x263) [0x1330463]
mongod(__wt_open_cursor+0xB1) [0x13637c1]
mongod(+0xF63BE5) [0x1363be5]
mongod(_ZN5mongo20WiredTigerSizeStorer8loadFromEPNS_17WiredTigerSessionERKSs+0x88) [0xd65388]
mongod(_ZN5mongo18WiredTigerKVEngineC2ERKSsS2_b+0x358) [0xd54708]
mongod(+0x952F30) [0xd52f30]
mongod(_ZN5mongo23GlobalEnvironmentMongoD22setGlobalStorageEngineERKSs+0x138) [0xa6a708]
mongod(_ZN5mongo13initAndListenEi+0x6EC) [0x7da5ec]
mongod(main+0x134) [0x7df304]
libc.so.6(__libc_start_main+0xF5) [0x7f62c3d98ec5]
mongod(+0x3D8259) [0x7d8259]
----- END BACKTRACE -----

Comment by Daniel Pasette (Inactive) [ 19/Nov/14 ]

Closing this as WAD.

Comment by Daniel Pasette (Inactive) [ 18/Nov/14 ]

asya, any further information on steps taken to get in this state?

Generated at Thu Feb 08 03:40:19 UTC 2024 using Jira 9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66.