[SERVER-45177] Not-equal ($ne) in query for mapReduce causes segmentation fault Created: 16/Dec/19  Updated: 29/Oct/23  Resolved: 21/Dec/19

Status: Closed
Project: Core Server
Component/s: MapReduce
Affects Version/s: 4.2.2
Fix Version/s: 4.2.3

Type: Bug Priority: Major - P3
Reporter: Pavel Belev Assignee: Charlie Swanson
Resolution: Fixed Votes: 2
Labels: qopt-team
Remaining Estimate: Not Specified
Time Spent: Not Specified
Original Estimate: Not Specified

Attachments: Text File mongod.log     File repro.js    
Issue Links:
Duplicate
is duplicated by SERVER-38776 Invalid access at address: 0 Got sig... Closed
is duplicated by SERVER-45629 server crashes whene mapReduce query ... Closed
Related
is related to NODE-2409 Server v4.2.2 segfaults when calling ... Closed
is related to SERVER-45281 Reduce danger of MatchExpression::get... Closed
Backwards Compatibility: Fully Compatible
Operating System: ALL
Steps To Reproduce:

 

Mongo shell repo steps with results:

> db.version()
4.2.2
> db.createCollection('test_map_ne')> db.createCollection('test_map_ne'){ "ok" : 1 }> db.test_map_ne.insert([{key: 'one', val: 1}, {key: 'two', val: 2}])BulkWriteResult({ "writeErrors" : [ ], "writeConcernErrors" : [ ], "nInserted" : 2, "nUpserted" : 0, "nMatched" : 0, "nModified" : 0, "nRemoved" : 0, "upserted" : [ ]})> db.test_map_ne.find({}){ "_id" : ObjectId("5df7c48074a7e8d75b9f2ea6"), "key" : "one", "val" : 1 }{ "_id" : ObjectId("5df7c48074a7e8d75b9f2ea7"), "key" : "two", "val" : 2 }> db.test_map_ne.find({key: {$ne: 'one'}}){ "_id" : ObjectId("5df7c48074a7e8d75b9f2ea7"), "key" : "two", "val" : 2 }> var mapper = function() {emit(this._id, this.val);};> var reducer = function(k, v) {return Array.sum(v);};> db.test_map_ne.mapReduce(mapper, reducer, {out: {inline: 1}, query: {}}){ "results" : [ { "_id" : ObjectId("5df7c48074a7e8d75b9f2ea6"), "value" : 1 }, { "_id" : ObjectId("5df7c48074a7e8d75b9f2ea7"), "value" : 2 } ], "timeMillis" : 24, "counts" : { "input" : 2, "emit" : 2, "reduce" : 0, "output" : 2 }, "ok" : 1}> db.test_map_ne.mapReduce(mapper, reducer, {out: {inline: 1}, query: {key: {$ne: 'one'}}})2019-12-16T19:56:10.707+0200 I  NETWORK  [js] DBClientConnection failed to receive message from 127.0.0.1:27017 - HostUnreachable: Connection closed by peer2019-12-16T19:56:10.707+0200 E  QUERY    [js] uncaught exception: Error: error doing query: failed: network error while attempting to run command 'mapreduce' on host '127.0.0.1:27017'  :DB.prototype.runCommand@src/mongo/shell/db.js:169:19DB.prototype.runReadCommand@src/mongo/shell/db.js:141:12DBCollection.prototype.mapReduce@src/mongo/shell/collection.js:1078:15@(shell):1:1

Backtrace:

----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"55E96C4B4000","o":"27FFA01","s":"_ZN5mongo15printStackTraceERSo"},{"b":"55E96C4B4000","o":"27FF1FE"},{"b":"55E96C4B4000","o":"27FF3DC"},{"b":"55E96C4B4000","o":"1DFFE74"},{"b":"7F74BB793000","o":"F5F0"},{"b":"55E96C4B4000","o":"123690C"},{"b":"55E96C4B4000","o":"12386B3","s":"_ZN5mongo2mr16MapReduceCommand9errmsgRunEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERS9_RNS_14BSONObjBuilderE"},{"b":"55E96C4B4000","o":"21FD276","s":"_ZN5mongo23ErrmsgCommandDeprecated3runEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERNS_14BSONObjBuilderE"},{"b":"55E96C4B4000","o":"2202DC4","s":"_ZN5mongo12BasicCommand10Invocation3runEPNS_16OperationContextEPNS_3rpc21ReplyBuilderInterfaceE"},{"b":"55E96C4B4000","o":"1146498"},{"b":"55E96C4B4000","o":"1148984"},{"b":"55E96C4B4000","o":"11496EA","s":"_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE"},{"b":"55E96C4B4000","o":"113731C","s":"_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE"},{"b":"55E96C4B4000","o":"114389C","s":"_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE"},{"b":"55E96C4B4000","o":"113F09F","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"55E96C4B4000","o":"114232C"},{"b":"55E96C4B4000","o":"1F45212","s":"_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE"},{"b":"55E96C4B4000","o":"113CA9D","s":"_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE"},{"b":"55E96C4B4000","o":"113FD53","s":"_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE"},{"b":"55E96C4B4000","o":"113E167","s":"_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE"},{"b":"55E96C4B4000","o":"113EFFB","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"55E96C4B4000","o":"114232C"},{"b":"55E96C4B4000","o":"1F4567B"},{"b":"55E96C4B4000","o":"2589454"},{"b":"7F74BB793000","o":"7E65"},{"b":"7F74BB3C5000","o":"FE88D","s":"clone"}],"processInfo":{ "mongodbVersion" : "4.2.2", "gitVersion" : "a0bbbff6ada159e19298d37946ac8dc4b497eadf", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.10.0-1062.4.3.el7.x86_64", "version" : "#1 SMP Wed Nov 13 23:58:53 UTC 2019", "machine" : "x86_64" }, "somap" : [ { "b" : "55E96C4B4000", "elfType" : 3, "buildId" : "67F0B211856C5980A144552C72D5392B3453718F" }, { "b" : "7FFE5D811000", "elfType" : 3, "buildId" : "9EAE8893CACF7C03FAC2CEED330551A226918061" }, { "b" : "7F74BCBC1000", "path" : "/lib64/libcurl.so.4", "elfType" : 3, "buildId" : "89C83CEB5DE5FDEC4F3DFBA4FCAACB53D747A998" }, { "b" : "7F74BC9A8000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "3009B26B33156EAAF99787AA3DA0C6AE99649755" }, { "b" : "7F74BC545000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "4CF1939F660008CFA869D8364651F31AACD2C1C4" }, { "b" : "7F74BC2D3000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "3B305C3BA17FE394862E749763F2956C9C890C2E" }, { "b" : "7F74BC0CF000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "18113E6E83D8E981B8E8D808F7F3DBB23F950A1D" }, { "b" : "7F74BBEC7000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "4749697BF078337576C4629F0D30B296A0939779" }, { "b" : "7F74BBBC5000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "5681C054FDABCF789F4DDA66E94F1F6ED1747327" }, { "b" : "7F74BB9AF000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "DAC0179F4555AEFEC9E97476201802FD20C03EC5" }, { "b" : "7F74BB793000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "8B33F7F8C86F8D544C63C5541A8E42B3DDFEF8B1" }, { "b" : "7F74BB3C5000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "398944D32CF16A67AF51067A326E6C0CC14F90ED" }, { "b" : "7F74BCE2B000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "5CC1A53B747A7E4D21198723C2B633E54F3C06D9" }, { "b" : "7F74BB192000", "path" : "/lib64/libidn.so.11", "elfType" : 3, "buildId" : "2B77BBEFFF65E94F3E0B71A4E89BEB68C4B476C5" }, { "b" : "7F74BAF65000", "path" : "/lib64/libssh2.so.1", "elfType" : 3, "buildId" : "1AF123CADB2F2910E89CBD540A06D3B33692F95E" }, { "b" : "7F74BAD0C000", "path" : "/lib64/libssl3.so", "elfType" : 3, "buildId" : "B6321C434B5C7386B144B925CEE2798D269FDDF5" }, { "b" : "7F74BAAE4000", "path" : "/lib64/libsmime3.so", "elfType" : 3, "buildId" : "BDA454441F59F41D2DA36E13CEA1FC4CE95B2BBB" }, { "b" : "7F74BA7B5000", "path" : "/lib64/libnss3.so", "elfType" : 3, "buildId" : "DC3B36B530F506DE4FC1A6612D7DF44D4A3DDCDB" }, { "b" : "7F74BA585000", "path" : "/lib64/libnssutil3.so", "elfType" : 3, "buildId" : "32C8FB6C2768FFE41E0A15CBF2089A4202CA2290" }, { "b" : "7F74BA381000", "path" : "/lib64/libplds4.so", "elfType" : 3, "buildId" : "325B8CE57A776DE0B24B362A7E0C90E903B1A4B8" }, { "b" : "7F74BA17C000", "path" : "/lib64/libplc4.so", "elfType" : 3, "buildId" : "0460FF10A3C63749113D380C40E10DFCF066C76E" }, { "b" : "7F74B9F3E000", "path" : "/lib64/libnspr4.so", "elfType" : 3, "buildId" : "8840B019EDB66B0CFBD2F77EF196440F7928106E" }, { "b" : "7F74B9CF1000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "E2AA8CA3D3164E7DBEC293BFA0B55D2B10DAC05D" }, { "b" : "7F74B9A08000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "3EE7267AF7BFD3B132E6A222D997DA09C96C90DD" }, { "b" : "7F74B97D5000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "82E28CACB60C27CD6F14A6D2268F0CFF621664D0" }, { "b" : "7F74B95D1000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "67E935BFABA2C914C01156B88947DD515EA51170" }, { "b" : "7F74B93C2000", "path" : "/lib64/liblber-2.4.so.2", "elfType" : 3, "buildId" : "3192C56CD451E18EB9F29CB045432BA9C738DD29" }, { "b" : "7F74B916D000", "path" : "/lib64/libldap-2.4.so.2", "elfType" : 3, "buildId" : "F1FADDDE0D21D5F4E2DCADEDD3B85B6E7AAC9883" }, { "b" : "7F74B8F57000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "B9D5F73428BD6AD68C96986B57BEA3B7CEDB9745" }, { "b" : "7F74B8D47000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "4F5FBB2087BE132892467C4E7A46A3D07E5DA40B" }, { "b" : "7F74B8B43000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "2E01D5AC08C1280D013AAB96B292AC58BC30A263" }, { "b" : "7F74B8926000", "path" : "/lib64/libsasl2.so.3", "elfType" : 3, "buildId" : "E2F2017F821DD1B9D307DA1A9B8014F2941AEB7B" }, { "b" : "7F74B86FF000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "D2DD4DA3FDE1477D25BFFF80F3A25FDB541A8179" }, { "b" : "7F74B84C8000", "path" : "/lib64/libcrypt.so.1", "elfType" : 3, "buildId" : "84467C988F41D853C58353BEB247670E15DA8BAD" }, { "b" : "7F74B8266000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "9CA3D11F018BEEB719CDB34BE800BF1641350D0A" }, { "b" : "7F74B8063000", "path" : "/lib64/libfreebl3.so", "elfType" : 3, "buildId" : "197680DAE6538245CB99723E57447C4EF2E98362" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x55e96ecb3a01]
 mongod(+0x27FF1FE) [0x55e96ecb31fe]
 mongod(+0x27FF3DC) [0x55e96ecb33dc]
 mongod(+0x1DFFE74) [0x55e96e2b3e74]
 libpthread.so.0(+0xF5F0) [0x7f74bb7a25f0]
 mongod(+0x123690C) [0x55e96d6ea90c]
 mongod(_ZN5mongo2mr16MapReduceCommand9errmsgRunEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERS9_RNS_14BSONObjBuilderE+0x6B3) [0x55e96d6ec6b3]
 mongod(_ZN5mongo23ErrmsgCommandDeprecated3runEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERNS_14BSONObjBuilderE+0x46) [0x55e96e6b1276]
 mongod(_ZN5mongo12BasicCommand10Invocation3runEPNS_16OperationContextEPNS_3rpc21ReplyBuilderInterfaceE+0x74) [0x55e96e6b6dc4]
 mongod(+0x1146498) [0x55e96d5fa498]
 mongod(+0x1148984) [0x55e96d5fc984]
 mongod(_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE+0x41A) [0x55e96d5fd6ea]
 mongod(_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE+0x3C) [0x55e96d5eb31c]
 mongod(_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE+0xEC) [0x55e96d5f789c]
 mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x17F) [0x55e96d5f309f]
 mongod(+0x114232C) [0x55e96d5f632c]
 mongod(_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE+0x182) [0x55e96e3f9212]
 mongod(_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE+0x10D) [0x55e96d5f0a9d]
 mongod(_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE+0x843) [0x55e96d5f3d53]
 mongod(_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE+0x2E7) [0x55e96d5f2167]
 mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0xDB) [0x55e96d5f2ffb]
 mongod(+0x114232C) [0x55e96d5f632c]
 mongod(+0x1F4567B) [0x55e96e3f967b]
 mongod(+0x2589454) [0x55e96ea3d454]
 libpthread.so.0(+0x7E65) [0x7f74bb79ae65]
 libc.so.6(clone+0x6D) [0x7f74bb4c388d]
-----  END BACKTRACE  -----

 

Sprint: Query 2019-12-30
Participants:
Case:

 Description   

When using mapReduce with a query with $ne in it the server crashes with segmentation fault.

 



 Comments   
Comment by Githook User [ 20/Dec/19 ]

Author:

{'name': 'Charlie Swanson', 'email': 'charlie.swanson@mongodb.com', 'username': 'cswanson310'}

Message: SERVER-45177 Avoid using getChildVector() to traverse MatchExpression
Branch: v4.2
https://github.com/mongodb/mongo/commit/74e2b22bcd9803532755eb0582e533fb9a12bce8

Comment by Charlie Swanson [ 20/Dec/19 ]

Code Review

Comment by Charlie Swanson [ 19/Dec/19 ]

Surprisingly, it looks like this was caused by this commit from SERVER-44348. I think the root cause is that MatchExpression::getChildVector() doesn't work for the NOT node:

expression_tree.h

    std::vector<MatchExpression*>* getChildVector() final {
        return nullptr;
    }

Given that, I think this patch will fix this issue, but it's worth looking into the history of getChildVector() and see whether we could improve this dangerous API.

diff --git a/src/mongo/db/commands/mr.cpp b/src/mongo/db/commands/mr.cpp
index 5edc6a426a..f11ed660b9 100644
--- a/src/mongo/db/commands/mr.cpp
+++ b/src/mongo/db/commands/mr.cpp
@@ -236,8 +236,8 @@ bool queryContainsMatchType(MatchExpression* root, MatchExpression::MatchType ma
     if (root->matchType() == match) {
         return false;
     } else if (root->numChildren() > 0) {
-        for (auto&& child : *(root->getChildVector())) {
-            if (!queryContainsMatchType(child, match)) {
+        for (size_t i = 0; i < root->numChildren(); i++) {
+            if (!queryContainsMatchType(root->getChild(i), match)) {
                 return false;
             }
         }

Leaving this in 'Needs Scheduling' so we can find someone who's free to investigate further.

Comment by Carl Champain (Inactive) [ 19/Dec/19 ]

pavel.belev@practicedent.com,

I'm passing this ticket along to the Query team for further investigation. Please keep watching it for updates.

Thanks again for pointing this out,
Carl

Comment by Pavel Belev [ 18/Dec/19 ]

Thank you Carl.

I have attached the log file.

It is from a clean Mongo 4.2.2, with the commands I pointed in the description ran in the mongo shell only.

db.version()
db.createCollection('test_map_ne')
db.test_map_ne.insert([{key: 'one', val: 1}, {key: 'two', val: 2}])
db.test_map_ne.find({})
db.test_map_ne.find({key: {$ne: 'one'}})
var mapper = function() {emit(this._id, this.val);};
var reducer = function(k, v) {return Array.sum(v);};
db.test_map_ne.mapReduce(mapper, reducer, {out: {inline: 1}, query: {}})
db.test_map_ne.mapReduce(mapper, reducer, {out: {inline: 1}, query: {key: {$ne: 'one'}}})

Comment by Carl Champain (Inactive) [ 18/Dec/19 ]

Hi pavel.belev@practicedent.com,

Thanks for the report.
We think we have reproduced this issue but would like to confirm this for certain. To help us, can you please provide the full mongod.log?

Kind regards,
Carl

 

Comment by Pavel Belev [ 16/Dec/19 ]

For some reason the repo steps got merged in a single line. Adding them here as normal text and not as code:

> db.version()
4.2.2
> db.createCollection('test_map_ne')
{ "ok" : 1 }
> db.test_map_ne.insert([
{key: 'one', val: 1},
{key: 'two', val: 2}
])
BulkWriteResult({
"writeErrors" : [ ],
"writeConcernErrors" : [ ],
"nInserted" : 2,
"nUpserted" : 0,
"nMatched" : 0,
"nModified" : 0,
"nRemoved" : 0,
"upserted" : [ ]
})
> db.test_map_ne.find({})
{ "_id" : ObjectId("5df7c48074a7e8d75b9f2ea6"), "key" : "one", "val" : 1 } 
{ "_id" : ObjectId("5df7c48074a7e8d75b9f2ea7"), "key" : "two", "val" : 2 }
> db.test_map_ne.find(\{key: {$ne: 'one'}})
{ "_id" : ObjectId("5df7c48074a7e8d75b9f2ea7"), "key" : "two", "val" : 2 }
> var mapper = function(){emit(this._id, this.val);};
> var reducer = function(k, v){return Array.sum(v);};
> db.test_map_ne.mapReduce(mapper, reducer, {out:{inline: 1}, query: {}})
{
"results" : [
 {"_id" : ObjectId("5df7c48074a7e8d75b9f2ea6"), "value" : 1 },
 { "_id" : ObjectId("5df7c48074a7e8d75b9f2ea7"), "value" : 2 }
],
"timeMillis" : 24,
"counts" :{ "input" : 2, "emit" : 2, "reduce" : 0, "output" : 2 },
"ok" : 1
}
> db.test_map_ne.mapReduce(mapper, reducer, {out:{inline: 1}, query: {key: {$ne: 'one'}}})
2019-12-16T19:56:10.707+0200 I NETWORK [js] DBClientConnection failed to receive message from 127.0.0.1:27017 - HostUnreachable: Connection closed by peer
2019-12-16T19:56:10.707+0200 E QUERY [js] uncaught exception: Error: error doing query: failed: network error while attempting to run command 'mapreduce' on host '127.0.0.1:27017' :
DB.prototype.runCommand@src/mongo/shell/db.js:169:19
DB.prototype.runReadCommand@src/mongo/shell/db.js:141:12
DBCollection.prototype.mapReduce@src/mongo/shell/collection.js:1078:15
@(shell):1:1
>

Generated at Thu Feb 08 05:08:06 UTC 2024 using Jira 9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66.