[SERVER-40101] Aborting Invariant failure issue with Mongodb Version 3.0.11 Created: 13/Mar/19  Updated: 13/Mar/19  Resolved: 13/Mar/19

Status: Closed
Project: Core Server
Component/s: Replication, WiredTiger
Affects Version/s: None
Fix Version/s: None

Type: Question Priority: Major - P3
Reporter: Karthick [X] Assignee: Eric Sedor
Resolution: Done Votes: 0
Labels: None
Remaining Estimate: Not Specified
Time Spent: Not Specified
Original Estimate: Not Specified

Participants:

 Description   

Hi Team,

Need your response here. 

In a 3 node replica set one of the node has crashed down with the below mentioned error : could you please let me know when this can happen and the reason for the same.

 

Logs :

mongod(_ZN5mongo15printStackTraceERSo+0x32) [0xfa7742]
mongod(ZN5mongo11DBException13traceIfNeededERKS0+0x12D) [0xf28fad]
mongod(_ZN5mongo22WriteConflictExceptionC1Ev+0x54) [0x9ff424]
mongod(_ZN5mongo17wtRCToStatus_slowEiPKc+0x32C) [0xdc9f3c]
mongod(ZN5mongo21WiredTigerIndexUnique7_insertEP11_wt_cursorRKNS_7BSONObjERKNS_8RecordIdEb+0x40F) [0xdadd6f]
mongod(_ZN5mongo15WiredTigerIndex6insertEPNS_16OperationContextERKNS_7BSONObjERKNS_8RecordIdEb+0xC4) [0xdad634]
mongod(_ZN5mongo22BtreeBasedAccessMethod6insertEPNS_16OperationContextERKNS_7BSONObjERKNS_8RecordIdERKNS_19InsertDeleteOptionsEPl+0x199) [0xab72f9]
mongod(_ZN5mongo12IndexCatalog12_indexRecordEPNS_16OperationContextEPNS_17IndexCatalogEntryERKNS_7BSONObjERKNS_8RecordIdE+0x72) [0x94ebb2]
mongod(_ZN5mongo12IndexCatalog11indexRecordEPNS_16OperationContextERKNS_7BSONObjERKNS_8RecordIdE+0x86) [0x94ef36]
mongod(_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0xB7) [0x93c307]
mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x8C) [0x93ce8c]
mongod(_ZN5mongo18WriteBatchExecutor13execOneInsertEPNS0_16ExecInsertsStateEPPNS_16WriteErrorDetailE+0xA71) [0x9dd7e1]
mongod(_ZN5mongo18WriteBatchExecutor11execInsertsERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_16WriteErrorDetailESaIS9_EE+0x31D) [0x9de59d]
mongod(_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaISE_EE+0x49) [0x9e0519]
mongod(_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x465) [0x9e0d45]
mongod(_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x17C) [0x9e394c]
mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x31) [0xa055d1]
mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC6C) [0xa065dc]
mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0xa072ab]
mongod(ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3+0x73B) [0xbd503b]
mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xABA) [0xae841a]
mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xF5) [0x835425]
mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x339) [0xf58c99]
libpthread.so.0(+0x7DC5) [0x7f6f74e55dc5]
libc.so.6(clone+0x6D) [0x7f6f73c0876d]

2019-03-12T11:08:00.448-0400 I - [WiredTigerRecordStoreThread for local.oplog.rs] Invariant failure: ret resulted in status UnknownError 16: Device or resource busy at src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp 520
2019-03-12T11:08:00.462-0400 I CONTROL [WiredTigerRecordStoreThread for local.oplog.rs]
0xfa7742 0xf449d9 0xf2a910 0xdb7db0 0xdbd786 0xf2b9bc 0xff5d2b 0x7f6f74e55dc5 0x7f6f73c0876d
----- BEGIN BACKTRACE -----

{"backtrace":[\{"b":"400000","o":"BA7742","s":"_ZN5mongo15printStackTraceERSo"}

,{"b":"400000","o":"B449D9","s":"_ZN5mongo10logContextEPKc"},{"b":"400000","o":"B2A910","s":"_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j"},{"b":"400000","o":"9B7DB0","s":"_ZN5mongo21WiredTigerRecordStore27cappedDeleteAsNeeded_inlockEPNS_16OperationContextERKNS_8RecordIdE"},{"b":"400000","o":"9BD786"},{"b":"400000","o":"B2B9BC","s":"_ZN5mongo13BackgroundJob7jobBodyEv"},{"b":"400000","o":"BF5D2B"},{"b":"7F6F74E4E000","o":"7DC5"},{"b":"7F6F73B11000","o":"F776D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.0.11", "gitVersion" : "48f8b49dc30cc2485c6c1f3db31b723258fcbf39", "uname" :

{ "sysname" : "Linux", "release" : "3.10.0-514.26.1.el7.x86_64", "version" : "#1 SMP Wed Jun 28 20:34:34 UTC 2017", "machine" : "x86_64" }

, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "AA236EAF128C42D86039B8A793B6A36E235B5DD6" }, { "b" : "7FFC1B8AF000", "elfType" : 3, "buildId" : "519248B90BB8B358DF6ACF399AB871BD9FF6F341" }, { "b" : "7F6F74E4E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "B940A174FDF8F7CD003B41C6E31742F849F5AA5B" }, { "b" : "7F6F74BE0000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "CC9D689FA242237FF00D4DB5236FCA89F51B266E" }, { "b" : "7F6F747F6000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "D2B005D20FCAFAE345088D325E791906236C4D09" }, { "b" : "7F6F745EE000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "1F5B7C155C8E05EF4D3FB23473FBE5DA9694F25F" }, { "b" : "7F6F743EA000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "4C58E37F4048B54D1939F45073E3A9337117FB31" }, { "b" : "7F6F740E8000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "AD698AD7878FB9B4ACF8AA6905F4BBA6C768A49B" }, { "b" : "7F6F73ED2000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "7A1D83E1C6432F6A5C6AA83C69C4A8AAF2DAFFCD" }, { "b" : "7F6F73B11000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "1F5C891A90024FC9AE7FF1A68A670776F1FA53C4" }, { "b" : "7F6F7506A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "33B12A250B16604E1E6594B7B53AE730BEFED0C2" }, { "b" : "7F6F738C3000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "2BA81D2CC17DB87340A221A5B24927C6982E9D11" }, { "b" : "7F6F735DC000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0766C945638A6B7FD57D1F0677C9D1507F958DAE" }, { "b" : "7F6F733D8000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "57A29598F648B93EB474023CC0022D3D2954EB12" }, { "b" : "7F6F731A6000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "2498A9FEE56010C8C19E34EE8021190493C65BAB" }, { "b" : "7F6F72F90000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "C5EFFBEF48CD7EE3C99AE4DAB46D41DF0D18B5A9" }, { "b" : "7F6F72D81000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "F2BA8A407853010352EA3F3F66E67F1CE0D31E8F" }, { "b" : "7F6F72B7D000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "7B32513C445FBAD9E8DCB984D0DABA6006F3844E" }, { "b" : "7F6F72963000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "1155EE8FAF00E4907D507EF9C2B3492409690BFB" }, { "b" : "7F6F7273C000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "2CAE9B3E96D60CEAEEC1DFEC42899B1522C0CD96" }, { "b" : "7F6F724DB000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "795697BAE92FF49AAFF56EAE2BB71A80219801C6" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x32) [0xfa7742]
mongod(_ZN5mongo10logContextEPKc+0xE9) [0xf449d9]
mongod(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0xE0) [0xf2a910]
mongod(_ZN5mongo21WiredTigerRecordStore27cappedDeleteAsNeeded_inlockEPNS_16OperationContextERKNS_8RecordIdE+0x8E0) [0xdb7db0]
mongod(+0x9BD786) [0xdbd786]
mongod(_ZN5mongo13BackgroundJob7jobBodyEv+0x12C) [0xf2b9bc]
mongod(+0xBF5D2B) [0xff5d2b]
libpthread.so.0(+0x7DC5) [0x7f6f74e55dc5]
libc.so.6(clone+0x6D) [0x7f6f73c0876d]
----- END BACKTRACE -----
2019-03-12T11:08:00.462-0400 I - [WiredTigerRecordStoreThread for local.oplog.rs]

 

 

 

 



 Comments   
Comment by Eric Sedor [ 13/Mar/19 ]

MongoDB 3.0 reached end of life on February of 2018. We are unfortunately unable to help investigate in the SERVER project, which is for bugs and feature requests in current versions. For assistance troubleshooting end of life versions, please post on the mongodb-user group or on Stack Overflow with the mongodb tag.

Generated at Thu Feb 08 04:54:01 UTC 2024 using Jira 9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66.