[SERVER-23633] WT_NOTFOUND causes assertion failure in WiredTigerIndexUnique::_insert Created: 11/Apr/16  Updated: 06/May/16  Resolved: 06/May/16

Status: Closed
Project: Core Server
Component/s: WiredTiger
Affects Version/s: 3.2.4
Fix Version/s: None

Type: Bug Priority: Critical - P2
Reporter: Bruce Lucas (Inactive) Assignee: Michael Cahill (Inactive)
Resolution: Cannot Reproduce Votes: 0
Labels: None
Remaining Estimate: Not Specified
Time Spent: Not Specified
Original Estimate: Not Specified

Issue Links:
Related
Operating System: ALL
Participants:

 Description   

Attached assertion failure was encountered on 3.2.4. It appears we've just inserted a key into a unique index, got a return code from WT indicating the key is already in the index, tried to read that existing entry, but then got a return code from WT that it was not found, which is unexpected, resulting in an assertion failure.

2016-04-08T07:57:37.986-0500 I -        [conn67598] Invariant failure: ret resulted in status UnknownError: -31803: WT_NOTFOUND: item not found at src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp 988
2016-04-08T07:57:38.040-0500 I CONTROL  [conn67598]
 0x1310252 0x12ac428 0x129888d 0x1069910 0x1069ca7 0xc889af 0xace769 0xad2b0f 0xad2bb0 0xab22b5 0xab2621 0xab27cb 0xbff18b 0xc003b4 0xe0dbf5 0xe0e2b9 0xe0e3b5 0xb76d99 0xb78ef5 0xb792bd 0xb7c4d8 0xb95fe3 0xb96e74 0xaee770 0xcab2e5 0x97bdcc 0x12bc06d 0x3aa24079d1 0x3aa20e88fd
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"F10252","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"EAC428","s":"_ZN5mongo10logContextEPKc"},{"b":"400000","o":"E9888D","s":"_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j"},{"b":"400000","o":"C69910","s":"_ZN5mongo21WiredTigerIndexUnique7_insertEP11__wt_cursorRKNS_7BSONObjERKNS_8RecordIdEb"},{"b":"400000","o":"C69CA7","s":"_ZN5mongo15WiredTigerIndex6insertEPNS_16OperationContextERKNS_7BSONObjERKNS_8RecordIdEb"},{"b":"400000","o":"8889AF","s":"_ZN5mongo17IndexAccessMethod6insertEPNS_16OperationContextERKNS_7BSONObjERKNS_8RecordIdERKNS_19InsertDeleteOptionsEPl"},{"b":"400000","o":"6CE769","s":"_ZN5mongo12IndexCatalog21_indexFilteredRecordsEPNS_16OperationContextEPNS_17IndexCatalogEntryERKSt6vectorINS_10BsonRecordESaIS6_EE"},{"b":"400000","o":"6D2B0F","s":"_ZN5mongo12IndexCatalog13_indexRecordsEPNS_16OperationContextEPNS_17IndexCatalogEntryERKSt6vectorINS_10BsonRecordESaIS6_EE"},{"b":"400000","o":"6D2BB0","s":"_ZN5mongo12IndexCatalog12indexRecordsEPNS_16OperationContextERKSt6vectorINS_10BsonRecordESaIS4_EE"},{"b":"400000","o":"6B22B5","s":"_ZN5mongo10Collection16_insertDocumentsEPNS_16OperationContextEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorIS5_SaIS5_EEEESB_b"},{"b":"400000","o":"6B2621","s":"_ZN5mongo10Collection15insertDocumentsEPNS_16OperationContextEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorIS5_SaIS5_EEEESB_bb"},{"b":"400000","o":"6B27CB","s":"_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEbb"},{"b":"400000","o":"7FF18B","s":"_ZN5mongo11UpdateStage8doInsertEv"},{"b":"400000","o":"8003B4","s":"_ZN5mongo11UpdateStage4workEPm"},{"b":"400000","o":"A0DBF5","s":"_ZN5mongo12PlanExecutor11getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE"},{"b":"400000","o":"A0E2B9","s":"_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE"},{"b":"400000","o":"A0E3B5","s":"_ZN5mongo12PlanExecutor11executePlanEv"},{"b":"400000","o":"776D99","s":"_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEPNS_7BSONObjEPPNS_16WriteErrorDetailE"},{"b":"400000","o":"778EF5","s":"_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestEPSt6vectorIPNS_19BatchedUpsertDetailESaIS6_EEPS4_IPNS_16WriteErrorDetailESaISB_EE"},{"b":"400000","o":"7792BD","s":"_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE"},{"b":"400000","o":"77C4D8","s":"_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderE"},{"b":"400000","o":"795FE3","s":"_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE"},{"b":"400000","o":"796E74","s":"_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE"},{"b":"400000","o":"6EE770","s":"_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE"},{"b":"400000","o":"8AB2E5","s":"_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE"},{"b":"400000","o":"57BDCC","s":"_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortE"},{"b":"400000","o":"EBC06D","s":"_ZN5mongo17PortMessageServer17handleIncomingMsgEPv"},{"b":"3AA2400000","o":"79D1"},{"b":"3AA2000000","o":"E88FD","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.2.4", "gitVersion" : "e2ee9ffcf9f5a94fad76802e28cc978718bb7a30", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "2.6.32-504.8.1.el6.x86_64", "version" : "#1 SMP Wed Jan 28 21:11:36 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "5FD49352786BCEAEC14C2C99263D7EDCC9F681FD" }, { "b" : "7FFF6F014000", "elfType" : 3, "buildId" : "71A96B79419FE40FF1706D59ED06D6DBE37C8E97" }, { "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "6AC11C021C127E84A2724F3A750461A2183B1069" }, { "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "78A423B7DD3C00F7A77027D8D764BB432B57F1E7" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "583411D8786F86A1D6B8741C502831E6122445A7" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "454F8FC6CC6502C6401E5F9E221564D80665D277" }, { "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "C03877A9EE01DDC572E2B0F55F64C757773CF8D6" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "7D8E9374F4A4EA38A7C1E763F32240EA113E4208" }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "9350579A4970FA47F3144AD8F40B183B0954497D" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "B8DFF8E53D9F2B80C3C382E83EC17C828B536A39" }, { "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "E4EAB3C200B7D8444FF95AB01F6466924A6A5F5F" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "6F8E59B70E469F3A924A268911FF8FD0C37E7460" }, { "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "441FA45097A11508E50D55A3D1FF169BF2BE7C62" }, { "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "F62622218875795666E08B92D176A50791183EEC" }, { "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "152E2C18A7A2145021A8A879A01A82EE134E3946" }, { "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "B8DEDADC140347276164C729418C7A37B7224135" }, { "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "4BDFC7A19C1F328EB4FCFBCE7A1E27606928610D" }, { "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F8B68F301C19BF06AF56B4B06E0A69F89D2C1F8D" }, { "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "E6798A06BEE17CF102BBA44FD512FF8B805CEAF1" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x1310252]
 mongod(_ZN5mongo10logContextEPKc+0x138) [0x12ac428]
 mongod(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0xAD) [0x129888d]
 mongod(_ZN5mongo21WiredTigerIndexUnique7_insertEP11__wt_cursorRKNS_7BSONObjERKNS_8RecordIdEb+0x4E0) [0x1069910]
 mongod(_ZN5mongo15WiredTigerIndex6insertEPNS_16OperationContextERKNS_7BSONObjERKNS_8RecordIdEb+0xC7) [0x1069ca7]
 mongod(_ZN5mongo17IndexAccessMethod6insertEPNS_16OperationContextERKNS_7BSONObjERKNS_8RecordIdERKNS_19InsertDeleteOptionsEPl+0x19F) [0xc889af]
 mongod(_ZN5mongo12IndexCatalog21_indexFilteredRecordsEPNS_16OperationContextEPNS_17IndexCatalogEntryERKSt6vectorINS_10BsonRecordESaIS6_EE+0x109) [0xace769]
 mongod(_ZN5mongo12IndexCatalog13_indexRecordsEPNS_16OperationContextEPNS_17IndexCatalogEntryERKSt6vectorINS_10BsonRecordESaIS6_EE+0x11F) [0xad2b0f]
 mongod(_ZN5mongo12IndexCatalog12indexRecordsEPNS_16OperationContextERKSt6vectorINS_10BsonRecordESaIS4_EE+0x80) [0xad2bb0]
 mongod(_ZN5mongo10Collection16_insertDocumentsEPNS_16OperationContextEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorIS5_SaIS5_EEEESB_b+0x325) [0xab22b5]
 mongod(_ZN5mongo10Collection15insertDocumentsEPNS_16OperationContextEN9__gnu_cxx17__normal_iteratorIPKNS_7BSONObjESt6vectorIS5_SaIS5_EEEESB_bb+0x1B1) [0xab2621]
 mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEbb+0x6B) [0xab27cb]
 mongod(_ZN5mongo11UpdateStage8doInsertEv+0x16B) [0xbff18b]
 mongod(_ZN5mongo11UpdateStage4workEPm+0xB4) [0xc003b4]
 mongod(_ZN5mongo12PlanExecutor11getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0x275) [0xe0dbf5]
 mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x39) [0xe0e2b9]
 mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x55) [0xe0e3b5]
 mongod(_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEPNS_7BSONObjEPPNS_16WriteErrorDetailE+0x6F9) [0xb76d99]
 mongod(_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestEPSt6vectorIPNS_19BatchedUpsertDetailESaIS6_EEPS4_IPNS_16WriteErrorDetailESaISB_EE+0x2B5) [0xb78ef5]
 mongod(_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x1DD) [0xb792bd]
 mongod(_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderE+0x248) [0xb7c4d8]
 mongod(_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE+0x473) [0xb95fe3]
 mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE+0x404) [0xb96e74]
 mongod(_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE+0x1F0) [0xaee770]
 mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB45) [0xcab2e5]
 mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortE+0xEC) [0x97bdcc]
 mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x26D) [0x12bc06d]
 libpthread.so.0(+0x79D1) [0x3aa24079d1]
 libc.so.6(clone+0x6D) [0x3aa20e88fd]
-----  END BACKTRACE  -----



 Comments   
Comment by Ramon Fernandez Marina [ 06/May/16 ]

Unfortunately we have not been able to reproduce this behavior after multiple attempts, so we're closing this ticket for now.

If you see this error message in your logs feel free to post a comment here and we can reopen the ticket to investigate further.

Thanks,
Ramón.

Generated at Thu Feb 08 04:04:00 UTC 2024 using Jira 9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66.