[SERVER-20512] Segfault in DeleteStage::work Created: 19/Sep/15  Updated: 15/Oct/15  Resolved: 07/Oct/15

Status: Closed
Project: Core Server
Component/s: Internal Client, Write Ops
Affects Version/s: None
Fix Version/s: 3.2.0-rc0

Type: Bug Priority: Major - P3
Reporter: Bruce Lucas (Inactive) Assignee: Max Hirschhorn
Resolution: Done Votes: 0
Labels: None
Remaining Estimate: Not Specified
Time Spent: Not Specified
Original Estimate: Not Specified

Issue Links:
Related
is related to SERVER-16444 Avoid copying data out of WT buffers ... Closed
Backwards Compatibility: Fully Compatible
Operating System: ALL
Sprint: QuInt A (10/12/15)
Participants:

 Description   

On a recent build from master, was running 8 mongo shell instances of the following:

function insert_benchrun() {
    docs = []
    for (var i=0; i<10000; i++)
        docs.push({x:0})
    ops = [{
        op: "insert",
        ns: "test.c",
        doc: docs,
        safe: true,
    }]
    res = benchRun({
        ops: ops,
        seconds: 10000,
        parallel: 1
    })
    printjson(res)
}

and 2 mongo shell instances each doing the following:

function remove(t) {
    while (true) {
        printjson([t, db.c.remove({})])
    }
}

Got the following segfault. Also observed via mongostat that the inserts were succeeding but no deletes were being done.

2015-09-19T08:29:40.077-0400 F -        [conn5] Invalid access at address: 0x104fbff4
2015-09-19T08:29:40.103-0400 F -        [conn5] Got signal: 11 (Segmentation fault).
 
 0x123a7a2 0x12398f9 0x1239c87 0x7fd7559af340 0x7fd755672005 0x9655c8 0x965641 0xbb02cc 0xb6e056 0xd9f22d 0xd9f899 0xd9f95d 0xb2817f 0xb29230 0xb29782 0xb2c6e5 0xb44c07 0xb45a20 0xaa093d 0xc4d90e 0x945fed 0x11f755d 0x7fd7559a7182 0x7fd7556d447d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"E3A7A2"},{"b":"400000","o":"E398F9"},{"b":"400000","o":"E39C87"},{"b":"7FD75599F000","o":"10340"},{"b":"7FD7555DA000","o":"98005"},{"b":"400000","o":"5655C8"},{"b":"400000","o":"565641"},{"b":"400000","o":"7B02CC"},{"b":"400000","o":"76E056"},{"b":"400000","o":"99F22D"},{"b":"400000","o":"99F899"},{"b":"400000","o":"99F95D"},{"b":"400000","o":"72817F"},{"b":"400000","o":"729230"},{"b":"400000","o":"729782"},{"b":"400000","o":"72C6E5"},{"b":"400000","o":"744C07"},{"b":"400000","o":"745A20"},{"b":"400000","o":"6A093D"},{"b":"400000","o":"84D90E"},{"b":"400000","o":"545FED"},{"b":"400000","o":"DF755D"},{"b":"7FD75599F000","o":"8182"},{"b":"7FD7555DA000","o":"FA47D"}],"processInfo":{ "mongodbVersion" : "3.1.9-pre-", "gitVersion" : "e61e8a9cbd3c5c1e5a46fc74f4b5ab5ce879c115", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.13.0-32-generic", "version" : "#57-Ubuntu SMP Tue Jul 15 03:51:08 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "F1C60041E5958103EAB8B03EA61A47F4AEE8A06D" }, { "b" : "7FFF668FE000", "elfType" : 3, "buildId" : "E464DBB7341B7B9E7874DC0619C5F429416E6AC6" }, { "b" : "7FD7565E1000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "92FCF41EFE012D6186E31A59AD05BDBB487769AB" }, { "b" : "7FD7563DD000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "C1AE4CB7195D337A77A3C689051DABAA3980CA0C" }, { "b" : "7FD7560D9000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3, "buildId" : "4BF6F7ADD8244AD86008E6BF40D90F8873892197" }, { "b" : "7FD755DD3000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "1D76B71E905CB867B27CEF230FCB20F01A3178F5" }, { "b" : "7FD755BBD000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "CC0D578C2E0D86237CA7B0CE8913261C506A629A" }, { "b" : "7FD75599F000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "9318E8AF0BFBE444731BB0461202EF57F7C39542" }, { "b" : "7FD7555DA000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "30C94DC66A1FE95180C3D68D2B89E576D5AE213C" }, { "b" : "7FD7567E9000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "9F00581AB3C73E3AEA35995A0C50D24D59A01D47" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x123a7a2]
 mongod(+0xE398F9) [0x12398f9]
 mongod(+0xE39C87) [0x1239c87]
 libpthread.so.0(+0x10340) [0x7fd7559af340]
 libc.so.6(+0x98005) [0x7fd755672005]
 mongod(_ZNK5mongo7BSONObj4copyEv+0x38) [0x9655c8]
 mongod(_ZNK5mongo7BSONObj8getOwnedEv+0x41) [0x965641]
 mongod(_ZN5mongo16WorkingSetMember20makeObjOwnedIfNeededEv+0x3C) [0xbb02cc]
 mongod(_ZN5mongo11DeleteStage4workEPm+0x5C6) [0xb6e056]
 mongod(_ZN5mongo12PlanExecutor11getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0x28D) [0xd9f22d]
 mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x39) [0xd9f899]
 mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x4D) [0xd9f95d]
 mongod(_ZN5mongo18WriteBatchExecutor10execRemoveERKNS_12BatchItemRefEPPNS_16WriteErrorDetailE+0x4FF) [0xb2817f]
 mongod(_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestEPSt6vectorIPNS_19BatchedUpsertDetailESaIS6_EEPS4_IPNS_16WriteErrorDetailESaISB_EE+0x120) [0xb29230]
 mongod(_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x1D2) [0xb29782]
 mongod(_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderE+0x235) [0xb2c6e5]
 mongod(_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE+0x277) [0xb44c07]
 mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE+0x410) [0xb45a20]
 mongod(_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE+0x1AD) [0xaa093d]
 mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB8E) [0xc4d90e]
 mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortE+0xDD) [0x945fed]
 mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x27D) [0x11f755d]
 libpthread.so.0(+0x8182) [0x7fd7559a7182]
 libc.so.6(clone+0x6D) [0x7fd7556d447d]



 Comments   
Comment by Githook User [ 07/Oct/15 ]

Author:

{u'username': u'visemet', u'name': u'Max Hirschhorn', u'email': u'max.hirschhorn@mongodb.com'}

Message: SERVER-20512 Remove invalid calls to makeObjOwnedIfNeeded().

Don't try and make the object owned in DeleteStage if
Collection::deleteDocument() throws a WriteConflictException because we
would have already called saveState() on the child stage, which could
have freed the memory underlying the WorkingSetMember.

Similarly for UpdateStage with Collection::updateDocument().
Branch: master
https://github.com/mongodb/mongo/commit/33167b814bb9d8962205a85fed34c9d88942db83

Comment by Max Hirschhorn [ 02/Oct/15 ]

SERVER-16444 made it so that when a WriteConflictException occurs during Collection::deleteDocument(), an attempt to copy the record is made in case we would need to use it when retrying. Doing so is both unnecessary and invalid because we would have made a copy already if we were going to return the deleted document (i.e. for findAndModify) prior to calling Collection::deleteDocument(). The segfault was triggered by a combination of a WriteConflictException during Collection::deleteDocument() after the page of memory holding the document was evicted from the WiredTiger cache. This is perfectly valid storage engine behavior because saveState() was called on the the child of the DeleteStage, which is allowed to relinquish any storage engine resources.

Note that a similar issue of incorrectly attempting to make a copy of the record when a WriteConflictException occurs exists in UpdateStage. However, it technically isn't an invalid memory access because the record is unconditionally made owned prior to calling Collection::updateDocument() for storage engines that support document-level concurrency.

Comment by Max Hirschhorn [ 19/Sep/15 ]

Output from ASan while running two inserters and two deleters:

==28574==ERROR: AddressSanitizer: heap-use-after-free on address 0x62d000075dbf at pc 0x0000017c3d3c bp 0x7f22d42960f0 sp 0x7f22d42958a8
READ of size 4 at 0x62d000075dbf thread T21
    #0 0x17c3d3b in __asan_memcpy (/home/maxh/debugging/mongo/mongod+0x17c3d3b)
    #1 0x185bfd3 in mongo::DataType::Handler<int, void>::unsafeLoad(int*, char const*, unsigned long*) /home/maxh/debugging/mongo/src/mongo/base/data_type.h:67:17
    #2 0x185b90c in void mongo::DataType::unsafeLoad<int>(int*, char const*, unsigned long*) /home/maxh/debugging/mongo/src/mongo/base/data_type.h:150:9
    #3 0x185b4e6 in mongo::DataType::Handler<mongo::LittleEndian<int>, void>::unsafeLoad(mongo::LittleEndian<int>*, char const*, unsigned long*) /home/maxh/debugging/mongo/src/mongo/base/data_type_endian.h:123:13
    #4 0x185b1dc in void mongo::DataType::unsafeLoad<mongo::LittleEndian<int> >(mongo::LittleEndian<int>*, char const*, unsigned long*) /home/maxh/debugging/mongo/src/mongo/base/data_type.h:150:9
    #5 0x185af18 in mongo::ConstDataView const& mongo::ConstDataView::read<mongo::LittleEndian<int> >(mongo::LittleEndian<int>*, unsigned long) const /home/maxh/debugging/mongo/src/mongo/base/data_view.h:51:9
    #6 0x185a7f5 in mongo::LittleEndian<int> mongo::ConstDataView::read<mongo::LittleEndian<int> >(unsigned long) const /home/maxh/debugging/mongo/src/mongo/base/data_view.h:60:9
    #7 0x185a276 in mongo::BSONObj::objsize() const /home/maxh/debugging/mongo/src/mongo/bson/bsonobj.h:323:16
    #8 0x19ed74e in mongo::BSONObj::copy() const /home/maxh/debugging/mongo/src/mongo/bson/bsonobj.cpp:79:69
    #9 0x19eda12 in mongo::BSONObj::getOwned() const /home/maxh/debugging/mongo/src/mongo/bson/bsonobj.cpp:87:12
    #10 0x336368a in mongo::WorkingSetMember::makeObjOwnedIfNeeded() /home/maxh/debugging/mongo/src/mongo/db/exec/working_set.cpp:174:22
    #11 0x2f31cbb in mongo::DeleteStage::work(unsigned long*) /home/maxh/debugging/mongo/src/mongo/db/exec/delete.cpp:195:13
    #12 0x47e4ebf in mongo::PlanExecutor::getNextImpl(mongo::Snapshotted<mongo::BSONObj>*, mongo::RecordId*) /home/maxh/debugging/mongo/src/mongo/db/query/plan_executor.cpp:393:38
    #13 0x47e2f97 in mongo::PlanExecutor::getNext(mongo::BSONObj*, mongo::RecordId*) /home/maxh/debugging/mongo/src/mongo/db/query/plan_executor.cpp:322:23
    #14 0x47e846c in mongo::PlanExecutor::executePlan() /home/maxh/debugging/mongo/src/mongo/db/query/plan_executor.cpp:525:17
    #15 0x2bb0502 in mongo::multiRemove(mongo::OperationContext*, mongo::BatchItemRef const&, mongo::(anonymous namespace)::WriteOpResult*) /home/maxh/debugging/mongo/src/mongo/db/commands/write_commands/batch_executor.cpp:1340:29
    #16 0x2ba166d in mongo::WriteBatchExecutor::execRemove(mongo::BatchItemRef const&, mongo::WriteErrorDetail**) /home/maxh/debugging/mongo/src/mongo/db/commands/write_commands/batch_executor.cpp:871:5
    #17 0x2b9bbd9 in mongo::WriteBatchExecutor::bulkExecute(mongo::BatchedCommandRequest const&, std::vector<mongo::BatchedUpsertDetail*, std::allocator<mongo::BatchedUpsertDetail*> >*, std::vector<mongo::WriteErrorDetail*, std::allocator<mongo::WriteErrorDetail*> >*) /home/maxh/debugging/mongo/src/mongo/db/commands/write_commands/batch_executor.cpp:738:13
    #18 0x2b97c82 in mongo::WriteBatchExecutor::executeBatch(mongo::BatchedCommandRequest const&, mongo::BatchedCommandResponse*) /home/maxh/debugging/mongo/src/mongo/db/commands/write_commands/batch_executor.cpp:321:5
    #19 0x2bfc77a in mongo::WriteCmd::run(mongo::OperationContext*, std::string const&, mongo::BSONObj&, int, std::string&, mongo::BSONObjBuilder&) /home/maxh/debugging/mongo/src/mongo/db/commands/write_commands/write_commands.cpp:144:5
    #20 0x2d4597c in mongo::Command::run(mongo::OperationContext*, mongo::rpc::RequestInterface const&, mongo::rpc::ReplyBuilderInterface*) /home/maxh/debugging/mongo/src/mongo/db/dbcommands.cpp:1393:19
    #21 0x2d41e40 in mongo::Command::execCommand(mongo::OperationContext*, mongo::Command*, mongo::rpc::RequestInterface const&, mongo::rpc::ReplyBuilderInterface*) /home/maxh/debugging/mongo/src/mongo/db/dbcommands.cpp:1306:18
    #22 0x2719e22 in mongo::runCommands(mongo::OperationContext*, mongo::rpc::RequestInterface const&, mongo::rpc::ReplyBuilderInterface*) /home/maxh/debugging/mongo/src/mongo/db/commands.cpp:495:9
    #23 0x397fbae in mongo::(anonymous namespace)::receivedRpc(mongo::OperationContext*, mongo::Client&, mongo::DbResponse&, mongo::Message&) /home/maxh/debugging/mongo/src/mongo/db/instance.cpp:293:9
    #24 0x39746c0 in mongo::assembleResponse(mongo::OperationContext*, mongo::Message&, mongo::DbResponse&, mongo::HostAndPort const&) /home/maxh/debugging/mongo/src/mongo/db/instance.cpp:522:9
    #25 0x18846a7 in mongo::MyMessageHandler::process(mongo::Message&, mongo::AbstractMessagingPort*) /home/maxh/debugging/mongo/src/mongo/db/db.cpp:168:17
    #26 0x74411aa in mongo::PortMessageServer::handleIncomingMsg(void*) /home/maxh/debugging/mongo/src/mongo/util/net/message_server_port.cpp:229:17
    #27 0x7f22e1d5e181 in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x8181)
    #28 0x7f22e187547c in clone (/lib/x86_64-linux-gnu/libc.so.6+0xfa47c)
 
0x62d000075dbf is located 31167 bytes inside of 32758-byte region [0x62d00006e400,0x62d0000763f6)
freed by thread T17 here:
    #0 0x17da562 in __interceptor_free (/home/maxh/debugging/mongo/mongod+0x17da562)
    #1 0xce1db09 in __wt_free_int /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/os_posix/os_alloc.c:264:2
    #2 0xc866964 in __wt_page_out /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/btree/bt_discard.c:129:3
    #3 0xc865537 in __wt_ref_out /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/btree/bt_discard.c:32:2
    #4 0xc9657db in __wt_split_rewrite /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/btree/bt_split.c:1462:2
    #5 0xccd1712 in __evict_page_dirty_update /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/evict/evict_page.c:262:3
    #6 0xcccc1d3 in __wt_evict /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/evict/evict_page.c:122:4
    #7 0xc9e7b0b in __wt_page_release_evict /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/include/btree.i:1138:13
    #8 0xc9e2e40 in __wt_page_release /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/include/btree.i:1200:2
    #9 0xc9e5337 in __wt_page_swap_func /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/include/btree.i:1250:2
    #10 0xc9e131a in __wt_tree_walk /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/btree/bt_walk.c:301:10
    #11 0xc7d5873 in __wt_btcur_prev /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/btree/bt_curprev.c:618:3
    #12 0xcb97f79 in __curfile_prev /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/cursor/cur_file.c:156:13
    #13 0x5fbc8b0 in mongo::WiredTigerRecordStore::Cursor::restore() /home/maxh/debugging/mongo/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp:519:19
    #14 0x2f006d7 in mongo::CollectionScan::doRestoreState() /home/maxh/debugging/mongo/src/mongo/db/exec/collection_scan.cpp:224:14
    #15 0x313fd45 in mongo::PlanStage::restoreState() /home/maxh/debugging/mongo/src/mongo/db/exec/plan_stage.cpp:52:5
    #16 0x2f32122 in mongo::DeleteStage::work(unsigned long*) /home/maxh/debugging/mongo/src/mongo/db/exec/delete.cpp:214:13
    #17 0x47e4ebf in mongo::PlanExecutor::getNextImpl(mongo::Snapshotted<mongo::BSONObj>*, mongo::RecordId*) /home/maxh/debugging/mongo/src/mongo/db/query/plan_executor.cpp:393:38
    #18 0x47e2f97 in mongo::PlanExecutor::getNext(mongo::BSONObj*, mongo::RecordId*) /home/maxh/debugging/mongo/src/mongo/db/query/plan_executor.cpp:322:23
    #19 0x47e846c in mongo::PlanExecutor::executePlan() /home/maxh/debugging/mongo/src/mongo/db/query/plan_executor.cpp:525:17
    #20 0x2bb0502 in mongo::multiRemove(mongo::OperationContext*, mongo::BatchItemRef const&, mongo::(anonymous namespace)::WriteOpResult*) /home/maxh/debugging/mongo/src/mongo/db/commands/write_commands/batch_executor.cpp:1340:29
    #21 0x2ba166d in mongo::WriteBatchExecutor::execRemove(mongo::BatchItemRef const&, mongo::WriteErrorDetail**) /home/maxh/debugging/mongo/src/mongo/db/commands/write_commands/batch_executor.cpp:871:5
    #22 0x2b9bbd9 in mongo::WriteBatchExecutor::bulkExecute(mongo::BatchedCommandRequest const&, std::vector<mongo::BatchedUpsertDetail*, std::allocator<mongo::BatchedUpsertDetail*> >*, std::vector<mongo::WriteErrorDetail*, std::allocator<mongo::WriteErrorDetail*> >*) /home/maxh/debugging/mongo/src/mongo/db/commands/write_commands/batch_executor.cpp:738:13
    #23 0x2b97c82 in mongo::WriteBatchExecutor::executeBatch(mongo::BatchedCommandRequest const&, mongo::BatchedCommandResponse*) /home/maxh/debugging/mongo/src/mongo/db/commands/write_commands/batch_executor.cpp:321:5
    #24 0x2bfc77a in mongo::WriteCmd::run(mongo::OperationContext*, std::string const&, mongo::BSONObj&, int, std::string&, mongo::BSONObjBuilder&) /home/maxh/debugging/mongo/src/mongo/db/commands/write_commands/write_commands.cpp:144:5
    #25 0x2d4597c in mongo::Command::run(mongo::OperationContext*, mongo::rpc::RequestInterface const&, mongo::rpc::ReplyBuilderInterface*) /home/maxh/debugging/mongo/src/mongo/db/dbcommands.cpp:1393:19
    #26 0x2d41e40 in mongo::Command::execCommand(mongo::OperationContext*, mongo::Command*, mongo::rpc::RequestInterface const&, mongo::rpc::ReplyBuilderInterface*) /home/maxh/debugging/mongo/src/mongo/db/dbcommands.cpp:1306:18
    #27 0x2719e22 in mongo::runCommands(mongo::OperationContext*, mongo::rpc::RequestInterface const&, mongo::rpc::ReplyBuilderInterface*) /home/maxh/debugging/mongo/src/mongo/db/commands.cpp:495:9
    #28 0x397fbae in mongo::(anonymous namespace)::receivedRpc(mongo::OperationContext*, mongo::Client&, mongo::DbResponse&, mongo::Message&) /home/maxh/debugging/mongo/src/mongo/db/instance.cpp:293:9
    #29 0x39746c0 in mongo::assembleResponse(mongo::OperationContext*, mongo::Message&, mongo::DbResponse&, mongo::HostAndPort const&) /home/maxh/debugging/mongo/src/mongo/db/instance.cpp:522:9
 
previously allocated by thread T21 here:
    #0 0x17dab85 in __interceptor_realloc (/home/maxh/debugging/mongo/mongod+0x17dab85)
    #1 0xce1b8e5 in __wt_realloc /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/os_posix/os_alloc.c:104:11
    #2 0xd077b0d in __wt_buf_grow_worker /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/support/scratch.c:48:4
    #3 0xc8a84e0 in __wt_buf_grow /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/include/buf.i:18:6
    #4 0xc8a19db in __wt_buf_initsize /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/include/buf.i:61:2
    #5 0xc89f17d in __wt_bt_read /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/btree/bt_io.c:92:3
    #6 0xc8db67f in __page_read /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/btree/bt_read.c:371:2
    #7 0xc8d77ac in __wt_page_in_func /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/btree/bt_read.c:461:4
    #8 0xc9e4f19 in __wt_page_swap_func /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/include/btree.i:1234:8
    #9 0xc9e131a in __wt_tree_walk /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/btree/bt_walk.c:301:10
    #10 0xc7ab500 in __wt_btcur_next /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/btree/bt_curnext.c:532:3
    #11 0xc80a71e in __wt_btcur_search_near /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/btree/bt_cursor.c:448:20
    #12 0xcb9c848 in __curfile_search_near /home/maxh/debugging/mongo/src/third_party/wiredtiger/src/cursor/cur_file.c:222:2
    #13 0x5fbc0d1 in mongo::WiredTigerRecordStore::Cursor::restore() /home/maxh/debugging/mongo/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp:497:19
    #14 0x2f006d7 in mongo::CollectionScan::doRestoreState() /home/maxh/debugging/mongo/src/mongo/db/exec/collection_scan.cpp:224:14
    #15 0x313fd45 in mongo::PlanStage::restoreState() /home/maxh/debugging/mongo/src/mongo/db/exec/plan_stage.cpp:52:5
    #16 0x2f32122 in mongo::DeleteStage::work(unsigned long*) /home/maxh/debugging/mongo/src/mongo/db/exec/delete.cpp:214:13
    #17 0x47e4ebf in mongo::PlanExecutor::getNextImpl(mongo::Snapshotted<mongo::BSONObj>*, mongo::RecordId*) /home/maxh/debugging/mongo/src/mongo/db/query/plan_executor.cpp:393:38
    #18 0x47e2f97 in mongo::PlanExecutor::getNext(mongo::BSONObj*, mongo::RecordId*) /home/maxh/debugging/mongo/src/mongo/db/query/plan_executor.cpp:322:23
    #19 0x47e846c in mongo::PlanExecutor::executePlan() /home/maxh/debugging/mongo/src/mongo/db/query/plan_executor.cpp:525:17
    #20 0x2bb0502 in mongo::multiRemove(mongo::OperationContext*, mongo::BatchItemRef const&, mongo::(anonymous namespace)::WriteOpResult*) /home/maxh/debugging/mongo/src/mongo/db/commands/write_commands/batch_executor.cpp:1340:29
    #21 0x2ba166d in mongo::WriteBatchExecutor::execRemove(mongo::BatchItemRef const&, mongo::WriteErrorDetail**) /home/maxh/debugging/mongo/src/mongo/db/commands/write_commands/batch_executor.cpp:871:5
    #22 0x2b9bbd9 in mongo::WriteBatchExecutor::bulkExecute(mongo::BatchedCommandRequest const&, std::vector<mongo::BatchedUpsertDetail*, std::allocator<mongo::BatchedUpsertDetail*> >*, std::vector<mongo::WriteErrorDetail*, std::allocator<mongo::WriteErrorDetail*> >*) /home/maxh/debugging/mongo/src/mongo/db/commands/write_commands/batch_executor.cpp:738:13
    #23 0x2b97c82 in mongo::WriteBatchExecutor::executeBatch(mongo::BatchedCommandRequest const&, mongo::BatchedCommandResponse*) /home/maxh/debugging/mongo/src/mongo/db/commands/write_commands/batch_executor.cpp:321:5
    #24 0x2bfc77a in mongo::WriteCmd::run(mongo::OperationContext*, std::string const&, mongo::BSONObj&, int, std::string&, mongo::BSONObjBuilder&) /home/maxh/debugging/mongo/src/mongo/db/commands/write_commands/write_commands.cpp:144:5
    #25 0x2d4597c in mongo::Command::run(mongo::OperationContext*, mongo::rpc::RequestInterface const&, mongo::rpc::ReplyBuilderInterface*) /home/maxh/debugging/mongo/src/mongo/db/dbcommands.cpp:1393:19
    #26 0x2d41e40 in mongo::Command::execCommand(mongo::OperationContext*, mongo::Command*, mongo::rpc::RequestInterface const&, mongo::rpc::ReplyBuilderInterface*) /home/maxh/debugging/mongo/src/mongo/db/dbcommands.cpp:1306:18
    #27 0x2719e22 in mongo::runCommands(mongo::OperationContext*, mongo::rpc::RequestInterface const&, mongo::rpc::ReplyBuilderInterface*) /home/maxh/debugging/mongo/src/mongo/db/commands.cpp:495:9
    #28 0x397fbae in mongo::(anonymous namespace)::receivedRpc(mongo::OperationContext*, mongo::Client&, mongo::DbResponse&, mongo::Message&) /home/maxh/debugging/mongo/src/mongo/db/instance.cpp:293:9
    #29 0x39746c0 in mongo::assembleResponse(mongo::OperationContext*, mongo::Message&, mongo::DbResponse&, mongo::HostAndPort const&) /home/maxh/debugging/mongo/src/mongo/db/instance.cpp:522:9
 
Thread T21 created by T0 here:
    #0 0x17c2b40 in __interceptor_pthread_create (/home/maxh/debugging/mongo/mongod+0x17c2b40)
    #1 0x743db80 in mongo::PortMessageServer::accepted(std::shared_ptr<mongo::Socket>, long long) /home/maxh/debugging/mongo/src/mongo/util/net/message_server_port.cpp:148:26
    #2 0x743ef4e in non-virtual thunk to mongo::PortMessageServer::accepted(std::shared_ptr<mongo::Socket>, long long) /home/maxh/debugging/mongo/src/mongo/util/net/message_server_port.cpp:107:18
    #3 0x73fc101 in mongo::Listener::initAndListen() /home/maxh/debugging/mongo/src/mongo/util/net/listen.cpp:351:13
    #4 0x743ca71 in mongo::PortMessageServer::run() /home/maxh/debugging/mongo/src/mongo/util/net/message_server_port.cpp:176:9
    #5 0x1801571 in mongo::_initAndListen(int) /home/maxh/debugging/mongo/src/mongo/db/db.cpp:599:5
    #6 0x17fadef in mongo::initAndListen(int) /home/maxh/debugging/mongo/src/mongo/db/db.cpp:604:9
    #7 0x1802f7b in mongoDbMain(int, char**, char**) /home/maxh/debugging/mongo/src/mongo/db/db.cpp:844:25
    #8 0x180206d in main /home/maxh/debugging/mongo/src/mongo/db/db.cpp:649:20
    #9 0x7f22e179cec4 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x21ec4)
 
Thread T17 created by T0 here:
    #0 0x17c2b40 in __interceptor_pthread_create (/home/maxh/debugging/mongo/mongod+0x17c2b40)
    #1 0x743db80 in mongo::PortMessageServer::accepted(std::shared_ptr<mongo::Socket>, long long) /home/maxh/debugging/mongo/src/mongo/util/net/message_server_port.cpp:148:26
    #2 0x743ef4e in non-virtual thunk to mongo::PortMessageServer::accepted(std::shared_ptr<mongo::Socket>, long long) /home/maxh/debugging/mongo/src/mongo/util/net/message_server_port.cpp:107:18
    #3 0x73fc101 in mongo::Listener::initAndListen() /home/maxh/debugging/mongo/src/mongo/util/net/listen.cpp:351:13
    #4 0x743ca71 in mongo::PortMessageServer::run() /home/maxh/debugging/mongo/src/mongo/util/net/message_server_port.cpp:176:9
    #5 0x1801571 in mongo::_initAndListen(int) /home/maxh/debugging/mongo/src/mongo/db/db.cpp:599:5
    #6 0x17fadef in mongo::initAndListen(int) /home/maxh/debugging/mongo/src/mongo/db/db.cpp:604:9
    #7 0x1802f7b in mongoDbMain(int, char**, char**) /home/maxh/debugging/mongo/src/mongo/db/db.cpp:844:25
    #8 0x180206d in main /home/maxh/debugging/mongo/src/mongo/db/db.cpp:649:20
    #9 0x7f22e179cec4 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x21ec4)
 
SUMMARY: AddressSanitizer: heap-use-after-free ??:0 __asan_memcpy
Shadow bytes around the buggy address:
  0x0c5a80006b60: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c5a80006b70: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c5a80006b80: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c5a80006b90: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c5a80006ba0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
=>0x0c5a80006bb0: fd fd fd fd fd fd fd[fd]fd fd fd fd fd fd fd fd
  0x0c5a80006bc0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c5a80006bd0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c5a80006be0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c5a80006bf0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c5a80006c00: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
Shadow byte legend (one shadow byte represents 8 application bytes):
  Addressable:           00
  Partially addressable: 01 02 03 04 05 06 07 
  Heap left redzone:       fa
  Heap right redzone:      fb
  Freed heap region:       fd
  Stack left redzone:      f1
  Stack mid redzone:       f2
  Stack right redzone:     f3
  Stack partial redzone:   f4
  Stack after return:      f5
  Stack use after scope:   f8
  Global redzone:          f9
  Global init order:       f6
  Poisoned by user:        f7
  Container overflow:      fc
  Array cookie:            ac
  Intra object redzone:    bb
  ASan internal:           fe
  Left alloca redzone:     ca
  Right alloca redzone:    cb
==28574==ABORTING


git version: 37b328aeda5696ff21c86b1258708fe5bede3a16

Generated at Thu Feb 08 03:54:26 UTC 2024 using Jira 9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66.