-
Type:
Bug
-
Resolution: Won't Fix
-
Priority:
Major - P3
-
None
-
Affects Version/s: None
-
Component/s: None
-
Storage Execution
-
ALL
-
45
jsCore failed on Ubuntu 14.04 (RocksDB)
Host: ec2-54-82-137-152.compute-1.amazonaws.com
Project: MongoDB (3.6)
Commit: diff: SERVER-32604 prevent adding a binary last-stable shard if the cluster is upgrading or downgrading
repair_database.js - Logs | History
Found a definite cause of failure for jstests/core/repair_database.js
2018-02-23T18:13:47.302000: Javascript StackTrace
Logs:
Error: error doing query: failed: network error while attempting to run command 'repairDatabase' on host 'localhost:20500' :
Stacktrace:
DB.prototype.runCommand@src/mongo/shell/db.js:168:1 DB.prototype.repairDatabase@src/mongo/shell/db.js:584:16 @jstests/core/repair_database.js:40:22
Here are some events that may help with triaging the failure
2018-02-23T18:13:47.231000: 2018-02-23T18:13:47.230+0000 I STORAGE [conn52] Repairing collection repairDB.a
Logs:
2018-02-23T18:13:47.230+0000 I STORAGE [conn52] Repairing collection repairDB.a
2018-02-23T18:13:47.231+0000 F - [conn52] Invariant failure slice.size() == sizeof(int64_t) src/mongo/db/modules/rocksdb/src/rocks_record_store.cpp 1008
2018-02-23T18:13:47.231+0000 F - [conn52]
***aborting after invariant() failure
2018-02-23T18:13:47.260+0000 F - [conn52] Got signal: 6 (Aborted).
0x24628e8 0x2461d2d 0x2462134 0x7f96a7f07330 0x7f96a7b64c37 0x7f96a7b68028 0xbf5dae 0xf1ef78 0xf24c35 0xf1ab2a 0xefe4bf 0xef7dcb 0x11878b7 0xf90728 0x1e10f06 0x1e12217 0x1e0f1bf 0xf50cbe 0xf52ebf 0xf53cf0 0xf5fcef 0xf5bb87 0xf5ec5a 0x1dd0042 0xf5aae5 0xf5cef2 0xf5d801 0xf5bbf5 0xf5ec5a 0x1dd05c1 0x23381b4 0x7f96a7eff184 0x7f96a7c2c03d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"20628E8","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"2061D2D"},{"b":"400000","o":"2062134"},{"b":"7F96A7EF7000","o":"10330"},{"b":"7F96A7B2E000","o":"36C37","s":"gsignal"},{"b":"7F96A7B2E000","o":"3A028","s":"abort"},{"b":"400000","o":"7F5DAE","s":"_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j"},{"b":"400000","o":"B1EF78"},{"b":"400000","o":"B24C35","s":"_ZN5mongo16RocksRecordStoreC1ENS_10StringDataES1_PN7rocksdb2DBEPNS_19RocksCounterManagerEPNS_22RocksDurabilityManagerEPNS_24RocksCompactionSchedulerENSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEbllPNS_14CappedCallbackE"},{"b":"400000","o":"B1AB2A","s":"_ZN5mongo11RocksEngine14getRecordStoreEPNS_16OperationContextENS_10StringDataES3_RKNS_17CollectionOptionsE"},{"b":"400000","o":"AFE4BF","s":"_ZN5mongo26KVDatabaseCatalogEntryBase14initCollectionEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEb"},{"b":"400000","o":"AF7DCB","s":"_ZN5mongo15KVStorageEngine17repairRecordStoreEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE"},{"b":"400000","o":"D878B7","s":"_ZN5mongo14repairDatabaseEPNS_16OperationContextEPNS_13StorageEngineERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEbb"},{"b":"400000","o":"B90728","s":"_ZN5mongo17CmdRepairDatabase9errmsgRunEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERS8_RNS_14BSONObjBuilderE"},{"b":"400000","o":"1A10F06","s":"_ZN5mongo23ErrmsgCommandDeprecated3runEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERNS_14BSONObjBuilderE"},{"b":"400000","o":"1A12217","s":"_ZN5mongo12BasicCommand11enhancedRunEPNS_16OperationContextERKNS_12OpMsgRequestERNS_14BSONObjBuilderE"},{"b":"400000","o":"1A0F1BF","s":"_ZN5mongo7Command9publicRunEPNS_16OperationContextERKNS_12OpMsgRequestERNS_14BSONObjBuilderE"},{"b":"400000","o":"B50CBE"},{"b":"400000","o":"B52EBF"},{"b":"400000","o":"B53CF0","s":"_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE"},{"b":"400000","o":"B5FCEF","s":"_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE"},{"b":"400000","o":"B5BB87","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"400000","o":"B5EC5A"},{"b":"400000","o":"19D0042","s":"_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsE"},{"b":"400000","o":"B5AAE5","s":"_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS0_9OwnershipE"},{"b":"400000","o":"B5CEF2","s":"_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE"},{"b":"400000","o":"B5D801","s":"_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE"},{"b":"400000","o":"B5BBF5","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"400000","o":"B5EC5A"},{"b":"400000","o":"19D05C1"},{"b":"400000","o":"1F381B4"},{"b":"7F96A7EF7000","o":"8184"},{"b":"7F96A7B2E000","o":"FE03D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.6.3-2-ge1cc0fb", "gitVersion" : "e1cc0fb45f6b5bbe596158258122d188667a43fe", "compiledModules" : [ "rocksdb" ], "uname" : { "sysname" : "Linux", "release" : "3.13.0-24-generic", "version" : "#46-Ubuntu SMP Thu Apr 10 19:11:08 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E2B68AC9F64C81DEB87F0DBA0BB1FD9E47624ED3" }, { "b" : "7FFFDB8FE000", "elfType" : 3, "buildId" : "6755FAD2CADACDF1667E5B57FF1EDFC28DD1C976" }, { "b" : "7F96A8E68000", "path" : "/lib/x86_64-linux-gnu/libz.so.1", "elfType" : 3, "buildId" : "7EE9BC42787A0F9A793BDBFCB0671FD12C52BC2A" }, { "b" : "7F96A8C58000", "path" : "/lib/x86_64-linux-gnu/libbz2.so.1.0", "elfType" : 3, "buildId" : "E1031DDBFFE20367E874B7093EEC0C8D9F3B43F6" }, { "b" : "7F96A8A3D000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "AD304AFCE6847F7A4D66D22853E87CCBF5A66966" }, { "b" : "7F96A8835000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "4F930712D3609C93E380E5BE5DF73E7AD273531C" }, { "b" : "7F96A8631000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "034D6A4EE9DCAB4A34ABD644345CBBB42DC63088" }, { "b" : "7F96A832B000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "300C7884CDEB5667BEA2357D2B8E7A76397562D6" }, { "b" : "7F96A8115000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "36311B4457710AE5578C4BF00791DED7359DBB92" }, { "b" : "7F96A7EF7000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "F64B8AD471FBA1B7A3A64EFB01551E694975E1F7" }, { "b" : "7F96A7B2E000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "D9A10B8EF90300628DD0A3A535106967714D7328" }, { "b" : "7F96A9081000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "2CA513EDC89C7BC06EC183D1A3A03CC0F606319C" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x28) [0x24628e8]
mongod(+0x2061D2D) [0x2461d2d]
mongod(+0x2062134) [0x2462134]
libpthread.so.0(+0x10330) [0x7f96a7f07330]
libc.so.6(gsignal+0x37) [0x7f96a7b64c37]
libc.so.6(abort+0x148) [0x7f96a7b68028]
mongod(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0x0) [0xbf5dae]
mongod(+0xB1EF78) [0xf1ef78]
mongod(_ZN5mongo16RocksRecordStoreC1ENS_10StringDataES1_PN7rocksdb2DBEPNS_19RocksCounterManagerEPNS_22RocksDurabilityManagerEPNS_24RocksCompactionSchedulerENSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEbllPNS_14CappedCallbackE+0xD05) [0xf24c35]
mongod(_ZN5mongo11RocksEngine14getRecordStoreEPNS_16OperationContextENS_10StringDataES3_RKNS_17CollectionOptionsE+0x82A) [0xf1ab2a]
mongod(_ZN5mongo26KVDatabaseCatalogEntryBase14initCollectionEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEb+0x2AF) [0xefe4bf]
mongod(_ZN5mongo15KVStorageEngine17repairRecordStoreEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE+0x1DB) [0xef7dcb]
mongod(_ZN5mongo14repairDatabaseEPNS_16OperationContextEPNS_13StorageEngineERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEbb+0x5F7) [0x11878b7]
mongod(_ZN5mongo17CmdRepairDatabase9errmsgRunEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERS8_RNS_14BSONObjBuilderE+0x298) [0xf90728]
mongod(_ZN5mongo23ErrmsgCommandDeprecated3runEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERNS_14BSONObjBuilderE+0x36) [0x1e10f06]
mongod(_ZN5mongo12BasicCommand11enhancedRunEPNS_16OperationContextERKNS_12OpMsgRequestERNS_14BSONObjBuilderE+0x67) [0x1e12217]
mongod(_ZN5mongo7Command9publicRunEPNS_16OperationContextERKNS_12OpMsgRequestERNS_14BSONObjBuilderE+0x1F) [0x1e0f1bf]
mongod(+0xB50CBE) [0xf50cbe]
mongod(+0xB52EBF) [0xf52ebf]
mongod(_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE+0x290) [0xf53cf0]
mongod(_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE+0x9F) [0xf5fcef]
mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x77) [0xf5bb87]
mongod(+0xB5EC5A) [0xf5ec5a]
mongod(_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsE+0x182) [0x1dd0042]
mongod(_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS0_9OwnershipE+0x125) [0xf5aae5]
mongod(_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE+0xAF2) [0xf5cef2]
mongod(_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE+0x201) [0xf5d801]
mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0xE5) [0xf5bbf5]
mongod(+0xB5EC5A) [0xf5ec5a]
mongod(+0x19D05C1) [0x1dd05c1]
mongod(+0x1F381B4) [0x23381b4]
libpthread.so.0(+0x8184) [0x7f96a7eff184]
libc.so.6(clone+0x6D) [0x7f96a7c2c03d]
----- END BACKTRACE -----
mongod on port 20500 was expected to be running in _do_teardown(), but wasn't. Exited with code -6.
JavaScript Backtrace Context
Frame 44: jstests/core/repair_database.js:40
repair_database.js
036 // 4 037 var myColl2 = mydb.b; 038 myColl.insert(doc); 039 myColl2.insert(doc); 040 >>> assert.commandWorked(mydb.repairDatabase()); 041 assert.docEq(doc, myColl.findOne({a: doc.a})); 042 assert.docEq(doc, myColl2.findOne({a: doc.a})); 043 044 // 5 045 var ldb = db.getSisterDB("local");