Hi Team,

Need your response here. 

In a 3 node replica set one of the node has crashed down with the below mentioned error : could you please let me know when this can happen and the reason for the same.

 

Logs :

mongod(_ZN5mongo15printStackTraceERSo+0x32) [0xfa7742]
mongod(ZN5mongo11DBException13traceIfNeededERKS0+0x12D) [0xf28fad]
mongod(_ZN5mongo22WriteConflictExceptionC1Ev+0x54) [0x9ff424]
mongod(_ZN5mongo17wtRCToStatus_slowEiPKc+0x32C) [0xdc9f3c]
mongod(ZN5mongo21WiredTigerIndexUnique7_insertEP11_wt_cursorRKNS_7BSONObjERKNS_8RecordIdEb+0x40F) [0xdadd6f]
mongod(_ZN5mongo15WiredTigerIndex6insertEPNS_16OperationContextERKNS_7BSONObjERKNS_8RecordIdEb+0xC4) [0xdad634]
mongod(_ZN5mongo22BtreeBasedAccessMethod6insertEPNS_16OperationContextERKNS_7BSONObjERKNS_8RecordIdERKNS_19InsertDeleteOptionsEPl+0x199) [0xab72f9]
mongod(_ZN5mongo12IndexCatalog12_indexRecordEPNS_16OperationContextEPNS_17IndexCatalogEntryERKNS_7BSONObjERKNS_8RecordIdE+0x72) [0x94ebb2]
mongod(_ZN5mongo12IndexCatalog11indexRecordEPNS_16OperationContextERKNS_7BSONObjERKNS_8RecordIdE+0x86) [0x94ef36]
mongod(_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0xB7) [0x93c307]
mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x8C) [0x93ce8c]
mongod(_ZN5mongo18WriteBatchExecutor13execOneInsertEPNS0_16ExecInsertsStateEPPNS_16WriteErrorDetailE+0xA71) [0x9dd7e1]
mongod(_ZN5mongo18WriteBatchExecutor11execInsertsERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_16WriteErrorDetailESaIS9_EE+0x31D) [0x9de59d]
mongod(_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaISE_EE+0x49) [0x9e0519]
mongod(_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x465) [0x9e0d45]
mongod(_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x17C) [0x9e394c]
mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x31) [0xa055d1]
mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC6C) [0xa065dc]
mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0xa072ab]
mongod(ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3+0x73B) [0xbd503b]
mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xABA) [0xae841a]
mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xF5) [0x835425]
mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x339) [0xf58c99]
libpthread.so.0(+0x7DC5) [0x7f6f74e55dc5]
libc.so.6(clone+0x6D) [0x7f6f73c0876d]

2019-03-12T11:08:00.448-0400 I - [WiredTigerRecordStoreThread for local.oplog.rs] Invariant failure: ret resulted in status UnknownError 16: Device or resource busy at src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp 520
2019-03-12T11:08:00.462-0400 I CONTROL [WiredTigerRecordStoreThread for local.oplog.rs]
0xfa7742 0xf449d9 0xf2a910 0xdb7db0 0xdbd786 0xf2b9bc 0xff5d2b 0x7f6f74e55dc5 0x7f6f73c0876d
----- BEGIN BACKTRACE -----

{"backtrace":[\{"b":"400000","o":"BA7742","s":"_ZN5mongo15printStackTraceERSo"}

,{"b":"400000","o":"B449D9","s":"_ZN5mongo10logContextEPKc"},{"b":"400000","o":"B2A910","s":"_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j"},{"b":"400000","o":"9B7DB0","s":"_ZN5mongo21WiredTigerRecordStore27cappedDeleteAsNeeded_inlockEPNS_16OperationContextERKNS_8RecordIdE"},{"b":"400000","o":"9BD786"},{"b":"400000","o":"B2B9BC","s":"_ZN5mongo13BackgroundJob7jobBodyEv"},{"b":"400000","o":"BF5D2B"},{"b":"7F6F74E4E000","o":"7DC5"},{"b":"7F6F73B11000","o":"F776D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.0.11", "gitVersion" : "48f8b49dc30cc2485c6c1f3db31b723258fcbf39", "uname" :

{ "sysname" : "Linux", "release" : "3.10.0-514.26.1.el7.x86_64", "version" : "#1 SMP Wed Jun 28 20:34:34 UTC 2017", "machine" : "x86_64" }

, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "AA236EAF128C42D86039B8A793B6A36E235B5DD6" }, { "b" : "7FFC1B8AF000", "elfType" : 3, "buildId" : "519248B90BB8B358DF6ACF399AB871BD9FF6F341" }, { "b" : "7F6F74E4E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "B940A174FDF8F7CD003B41C6E31742F849F5AA5B" }, { "b" : "7F6F74BE0000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "CC9D689FA242237FF00D4DB5236FCA89F51B266E" }, { "b" : "7F6F747F6000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "D2B005D20FCAFAE345088D325E791906236C4D09" }, { "b" : "7F6F745EE000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "1F5B7C155C8E05EF4D3FB23473FBE5DA9694F25F" }, { "b" : "7F6F743EA000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "4C58E37F4048B54D1939F45073E3A9337117FB31" }, { "b" : "7F6F740E8000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "AD698AD7878FB9B4ACF8AA6905F4BBA6C768A49B" }, { "b" : "7F6F73ED2000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "7A1D83E1C6432F6A5C6AA83C69C4A8AAF2DAFFCD" }, { "b" : "7F6F73B11000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "1F5C891A90024FC9AE7FF1A68A670776F1FA53C4" }, { "b" : "7F6F7506A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "33B12A250B16604E1E6594B7B53AE730BEFED0C2" }, { "b" : "7F6F738C3000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "2BA81D2CC17DB87340A221A5B24927C6982E9D11" }, { "b" : "7F6F735DC000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0766C945638A6B7FD57D1F0677C9D1507F958DAE" }, { "b" : "7F6F733D8000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "57A29598F648B93EB474023CC0022D3D2954EB12" }, { "b" : "7F6F731A6000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "2498A9FEE56010C8C19E34EE8021190493C65BAB" }, { "b" : "7F6F72F90000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "C5EFFBEF48CD7EE3C99AE4DAB46D41DF0D18B5A9" }, { "b" : "7F6F72D81000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "F2BA8A407853010352EA3F3F66E67F1CE0D31E8F" }, { "b" : "7F6F72B7D000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "7B32513C445FBAD9E8DCB984D0DABA6006F3844E" }, { "b" : "7F6F72963000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "1155EE8FAF00E4907D507EF9C2B3492409690BFB" }, { "b" : "7F6F7273C000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "2CAE9B3E96D60CEAEEC1DFEC42899B1522C0CD96" }, { "b" : "7F6F724DB000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "795697BAE92FF49AAFF56EAE2BB71A80219801C6" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x32) [0xfa7742]
mongod(_ZN5mongo10logContextEPKc+0xE9) [0xf449d9]
mongod(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0xE0) [0xf2a910]
mongod(_ZN5mongo21WiredTigerRecordStore27cappedDeleteAsNeeded_inlockEPNS_16OperationContextERKNS_8RecordIdE+0x8E0) [0xdb7db0]
mongod(+0x9BD786) [0xdbd786]
mongod(_ZN5mongo13BackgroundJob7jobBodyEv+0x12C) [0xf2b9bc]
mongod(+0xBF5D2B) [0xff5d2b]
libpthread.so.0(+0x7DC5) [0x7f6f74e55dc5]
libc.so.6(clone+0x6D) [0x7f6f73c0876d]
----- END BACKTRACE -----
2019-03-12T11:08:00.462-0400 I - [WiredTigerRecordStoreThread for local.oplog.rs]