Loading...

Type: Bug
Resolution: Cannot Reproduce
Priority: Major - P3
Fix Version/s: None
Affects Version/s: 2.8.0-rc2
Component/s: Storage
Labels:
- wiredtiger

Backwards Compatibility:
Fully Compatible
Operating System:
ALL

My WT primary was in a state of relatively poor performance. I shut it down using sudo kill PID. The process seemed to receive this message and initiated a clean shutdown.

On restart, the mongod process immediately died with the following:

2014-12-09T14:52:55.234+0000 I CONTROL  [initandlisten] MongoDB starting : pid=32663 port=27017 dbpath=/data/mmsqa_6 64-bit host=ip-10-167-172-82
2014-12-09T14:52:55.235+0000 I CONTROL  [initandlisten] db version v2.8.0-rc2
2014-12-09T14:52:55.235+0000 I CONTROL  [initandlisten] git version: f77869e01a6ed235695a46fad441c0dff15202f6
2014-12-09T14:52:55.235+0000 I CONTROL  [initandlisten] build info: Linux build19.nj1.10gen.cc 2.6.32-431.3.1.el6.x86_64 #1 SMP Fri Jan 3 21:39:27 UTC 2014 x86_64 BOOST_LIB_VER
SION=1_49
2014-12-09T14:52:55.235+0000 I CONTROL  [initandlisten] allocator: tcmalloc
2014-12-09T14:52:55.235+0000 I CONTROL  [initandlisten] options: { config: "/data/mmsqa_6/automation-mongod.conf", net: { port: 27017 }, processManagement: { fork: true }, repl
ication: { replSetName: "mmsqa" }, storage: { dbPath: "/data/mmsqa_6", engine: "wiredTiger" }, systemLog: { destination: "file", path: "/data/mmsqa_6/mongodb.log" } }
2014-12-09T14:52:55.236+0000 I STORAGE  [initandlisten] wiredtiger_open config: create,cache_size=3G,session_max=20000,extensions=[local=(entry=index_collator_extension)],stati
stics=(all),log=(enabled=true,archive=true,path=journal),checkpoint=(wait=60,log_size=2GB),
2014-12-09T14:52:58.683+0000 E STORAGE  [initandlisten] no cursor for uri: table:collection-583-4518534777035340103
2014-12-09T14:52:58.683+0000 I -        [initandlisten] Invariant failure c src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp 916
2014-12-09T14:52:58.693+0000 I CONTROL  [initandlisten] 
 0xf0cfa9 0xeb6d41 0xe9c3a2 0xd32ecc 0xd32ff7 0xd33ce3 0xd3401a 0xd2e352 0xcbd5ab 0xcc1296 0xd2d6bb 0xa5b591 0x7da70f 0x7df2f4 0x7f2ed3d3cec5 0x7d8469
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"B0CFA9"},{"b":"400000","o":"AB6D41"},{"b":"400000","o":"A9C3A2"},{"b":"400000","o":"932ECC"},{"b":"400000","o":"932FF7"},{"b":"400000","o":"933
CE3"},{"b":"400000","o":"93401A"},{"b":"400000","o":"92E352"},{"b":"400000","o":"8BD5AB"},{"b":"400000","o":"8C1296"},{"b":"400000","o":"92D6BB"},{"b":"400000","o":"65B591"},{"
b":"400000","o":"3DA70F"},{"b":"400000","o":"3DF2F4"},{"b":"7F2ED3D1B000","o":"21EC5"},{"b":"400000","o":"3D8469"}],"processInfo":{ "mongodbVersion" : "2.8.0-rc2", "gitVersion"
 : "f77869e01a6ed235695a46fad441c0dff15202f6", "uname" : { "sysname" : "Linux", "release" : "3.13.0-36-generic", "version" : "#63-Ubuntu SMP Wed Sep 3 21:30:07 UTC 2014", "mach
ine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000" }, { "b" : "7FFF828D5000", "elfType" : 3 }, { "b" : "7F2ED4D0D000", "path" : "/lib/x86_64-linux-gnu/libpthread.so
.0", "elfType" : 3 }, { "b" : "7F2ED4B05000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3 }, { "b" : "7F2ED4901000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2",
 "elfType" : 3 }, { "b" : "7F2ED45FD000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3 }, { "b" : "7F2ED42F7000", "path" : "/lib/x86_64-linux-gnu/libm.so.
6", "elfType" : 3 }, { "b" : "7F2ED40E1000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3 }, { "b" : "7F2ED3D1B000", "path" : "/lib/x86_64-linux-gnu/libc.so.6"
, "elfType" : 3 }, { "b" : "7F2ED4F2B000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3 } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf0cfa9]
 mongod(_ZN5mongo10logContextEPKc+0xE1) [0xeb6d41]
 mongod(_ZN5mongo15invariantFailedEPKcS1_j+0xB2) [0xe9c3a2]
 mongod(_ZN5mongo21WiredTigerRecordStore8Iterator7_locateERKNS_8RecordIdEb+0x16C) [0xd32ecc]
 mongod(_ZN5mongo21WiredTigerRecordStore8IteratorC2ERKS0_PNS_16OperationContextERKNS_8RecordIdERKNS_20CollectionScanParams9DirectionEb+0xA7) [0xd32ff7]
 mongod(_ZNK5mongo21WiredTigerRecordStore11getIteratorEPNS_16OperationContextERKNS_8RecordIdERKNS_20CollectionScanParams9DirectionE+0x43) [0xd33ce3]
 mongod(_ZN5mongo21WiredTigerRecordStoreC1EPNS_16OperationContextERKNS_10StringDataES5_bllPNS_28CappedDocumentDeleteCallbackEPNS_20WiredTigerSizeStorerE+0x2EA) [0xd3401a]
 mongod(_ZN5mongo18WiredTigerKVEngine14getRecordStoreEPNS_16OperationContextERKNS_10StringDataES5_RKNS_17CollectionOptionsE+0x132) [0xd2e352]
 mongod(_ZN5mongo22KVDatabaseCatalogEntry14initCollectionEPNS_16OperationContextERKSs+0xEB) [0xcbd5ab]
 mongod(_ZN5mongo15KVStorageEngineC1EPNS_8KVEngineERKNS_22KVStorageEngineOptionsE+0x6B6) [0xcc1296]
 mongod(+0x92D6BB) [0xd2d6bb]
 mongod(_ZN5mongo23GlobalEnvironmentMongoD22setGlobalStorageEngineERKSs+0x131) [0xa5b591]
 mongod(_ZN5mongo13initAndListenEi+0x6EF) [0x7da70f]
 mongod(main+0x134) [0x7df2f4]
 libc.so.6(__libc_start_main+0xF5) [0x7f2ed3d3cec5]
 mongod(+0x3D8469) [0x7d8469]
-----  END BACKTRACE  -----
2014-12-09T14:52:58.693+0000 I -        [initandlisten] 

***aborting after invariant() failure

2014-12-09T14:52:58.701+0000 F -        [initandlisten] Got signal: 6 (Aborted).

 0xf0cfa9 0xf0c912 0xf0cc96 0x7f2ed3d51c30 0x7f2ed3d51bb9 0x7f2ed3d54fc8 0xe9c3e7 0xd32ecc 0xd32ff7 0xd33ce3 0xd3401a 0xd2e352 0xcbd5ab 0xcc1296 0xd2d6bb 0xa5b591 0x7da70f 0x7df2f4 0x7f2ed3d3cec5 0x7d8469
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"B0CFA9"},{"b":"400000","o":"B0C912"},{"b":"400000","o":"B0CC96"},{"b":"7F2ED3D1B000","o":"36C30"},{"b":"7F2ED3D1B000","o":"36BB9"},{"b":"7F2ED3D1B000","o":"39FC8"},{"b":"400000","o":"A9C3E7"},{"b":"400000","o":"932ECC"},{"b":"400000","o":"932FF7"},{"b":"400000","o":"933CE3"},{"b":"400000","o":"93401A"},{"b":"400000","o":"92E352"},{"b":"400000","o":"8BD5AB"},{"b":"400000","o":"8C1296"},{"b":"400000","o":"92D6BB"},{"b":"400000","o":"65B591"},{"b":"400000","o":"3DA70F"},{"b":"400000","o":"3DF2F4"},{"b":"7F2ED3D1B000","o":"21EC5"},{"b":"400000","o":"3D8469"}],"processInfo":{ "mongodbVersion" : "2.8.0-rc2", "gitVersion" : "f77869e01a6ed235695a46fad441c0dff15202f6", "uname" : { "sysname" : "Linux", "release" : "3.13.0-36-generic", "version" : "#63-Ubuntu SMP Wed Sep 3 21:30:07 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000" }, { "b" : "7FFF828D5000", "elfType" : 3 }, { "b" : "7F2ED4D0D000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3 }, { "b" : "7F2ED4B05000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3 }, { "b" : "7F2ED4901000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3 }, { "b" : "7F2ED45FD000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3 }, { "b" : "7F2ED42F7000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3 }, { "b" : "7F2ED40E1000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3 }, { "b" : "7F2ED3D1B000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3 }, { "b" : "7F2ED4F2B000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3 } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf0cfa9]
 mongod(+0xB0C912) [0xf0c912]
 mongod(+0xB0CC96) [0xf0cc96]
 libc.so.6(+0x36C30) [0x7f2ed3d51c30]
 libc.so.6(gsignal+0x39) [0x7f2ed3d51bb9]
 libc.so.6(abort+0x148) [0x7f2ed3d54fc8]
 mongod(_ZN5mongo15invariantFailedEPKcS1_j+0xF7) [0xe9c3e7]
 mongod(_ZN5mongo21WiredTigerRecordStore8Iterator7_locateERKNS_8RecordIdEb+0x16C) [0xd32ecc]
 mongod(_ZN5mongo21WiredTigerRecordStore8IteratorC2ERKS0_PNS_16OperationContextERKNS_8RecordIdERKNS_20CollectionScanParams9DirectionEb+0xA7) [0xd32ff7]
 mongod(_ZNK5mongo21WiredTigerRecordStore11getIteratorEPNS_16OperationContextERKNS_8RecordIdERKNS_20CollectionScanParams9DirectionE+0x43) [0xd33ce3]
 mongod(_ZN5mongo21WiredTigerRecordStoreC1EPNS_16OperationContextERKNS_10StringDataES5_bllPNS_28CappedDocumentDeleteCallbackEPNS_20WiredTigerSizeStorerE+0x2EA) [0xd3401a]
 mongod(_ZN5mongo18WiredTigerKVEngine14getRecordStoreEPNS_16OperationContextERKNS_10StringDataES5_RKNS_17CollectionOptionsE+0x132) [0xd2e352]
 mongod(_ZN5mongo22KVDatabaseCatalogEntry14initCollectionEPNS_16OperationContextERKSs+0xEB) [0xcbd5ab]
 mongod(_ZN5mongo15KVStorageEngineC1EPNS_8KVEngineERKNS_22KVStorageEngineOptionsE+0x6B6) [0xcc1296]
 mongod(+0x92D6BB) [0xd2d6bb]
 mongod(_ZN5mongo23GlobalEnvironmentMongoD22setGlobalStorageEngineERKSs+0x131) [0xa5b591]
 mongod(_ZN5mongo13initAndListenEi+0x6EF) [0x7da70f]
 mongod(main+0x134) [0x7df2f4]
 libc.so.6(__libc_start_main+0xF5) [0x7f2ed3d3cec5]
 mongod(+0x3D8469) [0x7d8469]
-----  END BACKTRACE  -----

Please note that this node previously had a clean resync on 2.8.0rc2 from an MMAPV1 primary;

duplicates

SERVER-16351 Erroneous document returned in query (wt) (mms-dev)

Closed

Details

Description

Attachments

Issue Links

Activity

People

Dates