[SERVER-30943] Segmentation fault on attempt to access an invalidated BSON Object in JS scope Created: 04/Sep/17  Updated: 30/Oct/23  Resolved: 11/Sep/17

Status: Closed
Project: Core Server
Component/s: JavaScript
Affects Version/s: 3.2.16
Fix Version/s: 3.2.17

Type: Bug Priority: Critical - P2
Reporter: Aaron Wang Assignee: Jonathan Reams
Resolution: Fixed Votes: 0
Labels: None
Remaining Estimate: Not Specified
Time Spent: Not Specified
Original Estimate: Not Specified

Attachments: Text File from-starting-to-cash.log    
Issue Links:
Depends
Duplicate
duplicates SERVER-23191 Group command needs to make a copy of... Closed
Backwards Compatibility: Fully Compatible
Operating System: ALL
Steps To Reproduce:

not sure, we have a worker type service, which do a lot of jobs periodically in the background, if we start it, the mongod will crash in a few minutes, if we do not start it, the mongod won't crash

so I guess some operations the worker is doing triggered this bug, but it's quite difficult to narrow it down, we'll try, but at the same time, please help to analyze above logs

Participants:
Linked BF Score: 0

 Description   

mongod crashed, with these logs:

2017-09-05T02:56:26.201+0800 F -        [js] Invalid access at address: 0
2017-09-05T02:56:26.211+0800 F -        [js] Got signal: 11 (Segmentation fault).
 
 0x1556b32 0x1555ad9 0x15564b7 0x7f2074522390 0x1a329c1 0x14c22a8 0x14cdeb3 0x14cfaa1 0x14c1d8c 0x14c2e77 0x14cd472 0x14c13d0 0x149f68d 0x14c8377 0x14c93d7 0x14900b6 0x1d00200 0x7f20745186ba 0x7f207424e3dd
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"1156B32","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"1155AD9"},{"b":"400000","o":"11564B7"},{"b":"7F2074511000","o":"11390"},{"b":"400000","o":"16329C1","s":"_Z16JS_IdArrayLengthP9JSContextP9JSIdArray"},{"b":"400000","o":"10C22A8","s":"_ZN5mongo5mozjs13ObjectWrapper24WriteFieldRecursionFrameC2EP9JSContextP8JSObjectPNS_14BSONObjBuilderENS_10StringDataE"},{"b":"400000","o":"10CDEB3","s":"_ZN5mongo5mozjs11ValueWriter12_writeObjectEPNS_14BSONObjBuilderENS_10StringDataEPNS0_13LifetimeStackINS0_13ObjectWrapper24WriteFieldRecursionFrameELm150EEE"},{"b":"400000","o":"10CFAA1","s":"_ZN5mongo5mozjs11ValueWriter9writeThisEPNS_14BSONObjBuilderENS_10StringDataEPNS0_13LifetimeStackINS0_13ObjectWrapper24WriteFieldRecursionFrameELm150EEE"},{"b":"400000","o":"10C1D8C","s":"_ZN5mongo5mozjs13ObjectWrapper11_writeFieldEPNS_14BSONObjBuilderENS1_3KeyEPNS0_13LifetimeStackINS1_24WriteFieldRecursionFrameELm150EEEPNS_7BSONObjE"},{"b":"400000","o":"10C2E77","s":"_ZN5mongo5mozjs13ObjectWrapper6toBSONEv"},{"b":"400000","o":"10CD472","s":"_ZN5mongo5mozjs11ValueWriter6toBSONEv"},{"b":"400000","o":"10C13D0","s":"_ZN5mongo5mozjs13ObjectWrapper9getObjectENS1_3KeyE"},{"b":"400000","o":"109F68D","s":"_ZN5mongo5mozjs14MozJSImplScope9getObjectEPKc"},{"b":"400000","o":"10C8377"},{"b":"400000","o":"10C93D7","s":"_ZN5mongo5mozjs15MozJSProxyScope10implThreadEPv"},{"b":"400000","o":"10900B6","s":"_ZN4nspr6Thread13ThreadRoutineEPv"},{"b":"400000","o":"1900200"},{"b":"7F2074511000","o":"76BA"},{"b":"7F2074147000","o":"1073DD","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.2.16", "gitVersion" : "056bf45128114e44c5358c7a8776fb582363e094", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "4.4.0-85-generic", "version" : "#108-Ubuntu SMP Mon Jul 3 17:23:59 UTC 2017", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "B4C77D1B42936B23E28A2739927CB25274DB2D96" }, { "b" : "7FFE3F9B3000", "elfType" : 3, "buildId" : "D15ADFEB8025A8E672717AE54C85898EEA5C9A89" }, { "b" : "7F207549D000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "675F454AD6FD0B6CA2E41127C7B98079DA37F7B6" }, { "b" : "7F2075059000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "2DA08A7E5BF610030DD33B70DB951399626B7496" }, { "b" : "7F2074E51000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "F951C1E0765FCAE48F82CAFE35D1ADD36D6C9AF9" }, { "b" : "7F2074C4D000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "0FC788F0861846257B5F1773FBD438E95DFC1032" }, { "b" : "7F2074944000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "FF7A33D389E756CA381A8189291A968EA5E1F4F8" }, { "b" : "7F207472E000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "68220AE2C65D65C1B6AAA12FA6765A6EC2F5F434" }, { "b" : "7F2074511000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "27F189EF8DB8C3734C6A678E6EF3CB0B206D58B2" }, { "b" : "7F2074147000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "088A6E00A1814622219F346B41E775B8DD46C518" }, { "b" : "7F2075706000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "9157F205547F0EB588E2AB1F2F120B74253A43EA" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x1556b32]
 mongod(+0x1155AD9) [0x1555ad9]
 mongod(+0x11564B7) [0x15564b7]
 libpthread.so.0(+0x11390) [0x7f2074522390]
 mongod(_Z16JS_IdArrayLengthP9JSContextP9JSIdArray+0x1) [0x1a329c1]
 mongod(_ZN5mongo5mozjs13ObjectWrapper24WriteFieldRecursionFrameC2EP9JSContextP8JSObjectPNS_14BSONObjBuilderENS_10StringDataE+0x388) [0x14c22a8]
 mongod(_ZN5mongo5mozjs11ValueWriter12_writeObjectEPNS_14BSONObjBuilderENS_10StringDataEPNS0_13LifetimeStackINS0_13ObjectWrapper24WriteFieldRecursionFrameELm150EEE+0x1F3) [0x14cdeb3]
 mongod(_ZN5mongo5mozjs11ValueWriter9writeThisEPNS_14BSONObjBuilderENS_10StringDataEPNS0_13LifetimeStackINS0_13ObjectWrapper24WriteFieldRecursionFrameELm150EEE+0x591) [0x14cfaa1]
 mongod(_ZN5mongo5mozjs13ObjectWrapper11_writeFieldEPNS_14BSONObjBuilderENS1_3KeyEPNS0_13LifetimeStackINS1_24WriteFieldRecursionFrameELm150EEEPNS_7BSONObjE+0x10C) [0x14c1d8c]
 mongod(_ZN5mongo5mozjs13ObjectWrapper6toBSONEv+0x357) [0x14c2e77]
 mongod(_ZN5mongo5mozjs11ValueWriter6toBSONEv+0x92) [0x14cd472]
 mongod(_ZN5mongo5mozjs13ObjectWrapper9getObjectENS1_3KeyE+0x70) [0x14c13d0]
 mongod(_ZN5mongo5mozjs14MozJSImplScope9getObjectEPKc+0x7D) [0x149f68d]
 mongod(+0x10C8377) [0x14c8377]
 mongod(_ZN5mongo5mozjs15MozJSProxyScope10implThreadEPv+0xE7) [0x14c93d7]
 mongod(_ZN4nspr6Thread13ThreadRoutineEPv+0x26) [0x14900b6]
 mongod(+0x1900200) [0x1d00200]
 libpthread.so.0(+0x76BA) [0x7f20745186ba]
 libc.so.6(clone+0x6D) [0x7f207424e3dd]
-----  END BACKTRACE  -----
2017-09-05T02:56:26.211+0800 F -        [js] /proc/self/maps:
00400000-021fc000 r-xp 00000000 fd:01 409776                             /usr/bin/mongod
2017-09-05T02:56:26.211+0800 F -        [js] 021fc000-022ab000 rw-p 01dfc000 fd:01 409776                             /usr/bin/mongod
2017-09-05T02:56:26.211+0800 F -        [js] 022ab000-02317000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 0404a000-854e4000 rw-p 00000000 00:00 0                                  [heap]
2017-09-05T02:56:26.211+0800 F -        [js] 7f20441e4000-7f20441e5000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20441e5000-7f20442e5000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20442e5000-7f20442e6000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20442e6000-7f20443e6000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20443e6000-7f20443e7000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20443e7000-7f20444e7000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20444e7000-7f20444e8000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20444e8000-7f20445e8000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20445e8000-7f20445e9000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20445e9000-7f20446e9000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20446e9000-7f20446ea000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20446ea000-7f20447ea000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20447ea000-7f20447eb000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20447eb000-7f20448eb000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20448eb000-7f20448ec000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20448ec000-7f20449ec000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20449ec000-7f20449ed000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20449ed000-7f2044aed000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f2044aed000-7f2044aee000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f2044aee000-7f2044bee000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f2044bee000-7f2044bef000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f2044bef000-7f2044cef000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f2044cef000-7f2044cf0000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f2044cf0000-7f2044df0000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f2044df0000-7f2044df1000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f2044df1000-7f2044ef1000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f2044ef1000-7f2044ef2000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f2044ef2000-7f2044ff2000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f2044ff2000-7f2044ff3000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f2044ff3000-7f20450f3000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20450f3000-7f20450f4000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20450f4000-7f20451f4000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20451f4000-7f20451f5000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20451f5000-7f20452f5000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20452f5000-7f20452f6000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20452f6000-7f20453f6000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20453f6000-7f20453f7000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20453f7000-7f20454f7000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20454f7000-7f20454f8000 ---p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20454f8000-7f20455f8000 rw-p 00000000 00:00 0
2017-09-05T02:56:26.211+0800 F -        [js] 7f20455f8000-7f20455f9000 ---p 00000000 00:00 0



 Comments   
Comment by Jonathan Reams [ 11/Sep/17 ]

This will be in the next release of 3.2. 3.4 should not be effected by this bug.

Comment by Ramon Fernandez Marina [ 08/Sep/17 ]

Author:

{'username': u'jbreams', 'name': u'Jonathan Reams', 'email': u'jbreams@mongodb.com'}

Message:SERVER-30943 Check return value of JS_Enumerate in ObjectWrapper/ValueWriter
Branch:v3.2
https://github.com/mongodb/mongo/commit/8c0fd02d2dc44e3addca3e4ea9dd7292ea8b3e6f

Comment by Jonathan Reams [ 06/Sep/17 ]

I think I have a minimum reproducible test case:

(function() {
var t = db.testcoll;
t.drop();
t.insert([
  {
    "_id" : ObjectId("59b0541f3b129f1abc9e00ab"),
    "name" : {"flushrouterconfig" : "a", "last" : "A"}
  },
  {"_id" : ObjectId("59b0541f3b129f1abc9e00ac"), "name" : "alice", "foo" : 1},
]);
 
Mongo = t.group({
  key : {foo : 1},
  initial : {count : 0, values : []},
  reduce : function(obj, prev) {
    prev.count++;
    prev.values.push(obj.name);
  }
});
})();

The crash is because SERVER-22053 was not backported to 3.2 quite right and doesn't check the result of JS_Enumerate() when converting from JSON to BSON, and is compounded by not backporting SERVER-23191.

Comment by Kelsey Schubert [ 05/Sep/17 ]

Thank you for the report, inetfuture. We're investigating and will let update this ticket when we know more.

Kind regards,
Kelsey

Comment by Aaron Wang [ 04/Sep/17 ]

from-starting-to-cash.log this contains logs from starting to crash, I omitted replset peer connecting errors(Connecting, failed to connect, Dropping all pooled connections, error in heartbeat, etc), because there are a log and I don't think it's related

Generated at Thu Feb 08 04:25:31 UTC 2024 using Jira 9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66.