|
Hello Team,
We are using the Stable version of mongo 4.4.9 and there is 4 node replica cluster, in that 1-Primary, 2-Secondary, and 1-Delayed node.
We are also facing the same error on the secondary MongoDB instance.
Below is our error log for the same.
{"t":{"$date":"2021-11-11T10:00:26.261-08:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn6952","msg":"Connection ended","attr":{"remote":"127.0.0.1:57556","connectionId":6952,"connectionCount":26}}
|
{"t":{"$date":"2021-11-11T10:00:26.261-08:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn6953","msg":"Connection ended","attr":{"remote":"xx.xx.xx.xx:38144","connectionId":6953,"connectionCount":25}}
|
{"t":{"$date":"2021-11-11T10:00:27.583-08:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"Connection accepted","attr":{"remote":"xx.xx.xx.xx:40716","connectionId":6954,"connectionCount":26}}
|
{"t":{"$date":"2021-11-11T10:00:27.583-08:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn6954","msg":"client metadata","attr":{"remote":"xx.xx.xx.xx:40716","client":"conn6954","doc":{"os":{"type":"linux","architecture":"amd64"},"driver":{"name":"mgo","version":"globalsign"}}}}
|
{"t":{"$date":"2021-11-11T10:00:27.593-08:00"},"s":"I", "c":"ACCESS", "id":20250, "ctx":"conn6954","msg":"Authentication succeeded","attr":{"mechanism":"SCRAM-SHA-1","speculative":false,"principalName":"newrelic","authenticationDatabase":"admin","remote":"xx.xx.xx.xx:40716","extraInfo":{}}}
|
{"t":{"$date":"2021-11-11T10:00:27.753-08:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn6954","msg":"Connection ended","attr":{"remote":"xx.xx.xx.xx:40716","connectionId":6954,"connectionCount":25}}
|
{"t":{"$date":"2021-11-11T10:00:30.868-08:00"},"s":"F", "c":"REPL", "id":21238, "ctx":"ReplWriterWorker-33","msg":"Writer worker caught exception","attr":{"error":"NamespaceNotFound: Failed to apply operation: { lsid: { id: UUID(\"82e67257-6cc3-4f85-b533-39d6db763148\"), uid: BinData(0, 26CF2A8F3157EDFB202FDC538950D84EFA3F537EE4A79935780F2075604FB508) }, txnNumber: 1, op: \"i\", ns: \"fpid.events\", ui: UUID(\"587beef0-6621-4298-994e-d472cb075ea7\"), o: { _id: ObjectId('618d5a3e4e14855a5232838a'), processed: 0.0, msg: \"did=+xxxxxxxx;voip_id=xxxx;action=test\", type: \"agent_flagged\", created_date: 1636653630.0 }, ts: Timestamp(1636653630, 3), t: 11, v: 2, wall: new Date(1636653630866), stmtId: 0, prevOpTime: { ts: Timestamp(0, 0), t: -1 } } :: caused by :: Unable to resolve 587beef0-6621-4298-994e-d472cb075ea7","oplogEntry":{"lsid":{"id":{"$uuid":"82e67257-6cc3-4f85-b533-39d6db763148"},"uid":{"$binary":{"base64":"Js8qjzFX7fsgL9xTiVDYTvo/U37kp5k1eA8gdWBPtQg=","subType":"0"}}},"txnNumber":1,"op":"i","ns":"fpid.events","ui":{"$uuid":"587beef0-6621-4298-994e-d472cb075ea7"},"o":{"_id":{"$oid":"618d5a3e4e14855a5232838a"},"processed":0.0,"msg":"did=+xxxxxxxx;voip_id=xxxx;action=test","type":"agent_flagged","created_date":1636653630.0},"ts":{"$timestamp":{"t":1636653630,"i":3}},"t":11,"v":2,"wall":{"$date":"2021-11-11T18:00:30.866Z"},"stmtId":0,"prevOpTime":{"ts":{"$timestamp":{"t":0,"i":0}},"t":-1}}}}
|
{"t":{"$date":"2021-11-11T10:00:30.868-08:00"},"s":"F", "c":"REPL", "id":21235, "ctx":"OplogApplier-0","msg":"Failed to apply batch of operations","attr":{"numOperationsInBatch":1,"firstOperation":{"lsid":{"id":{"$uuid":"82e67257-6cc3-4f85-b533-39d6db763148"},"uid":{"$binary":{"base64":"Js8qjzFX7fsgL9xTiVDYTvo/U37kp5k1eA8gdWBPtQg=","subType":"0"}}},"txnNumber":1,"op":"i","ns":"fpid.events","ui":{"$uuid":"587beef0-6621-4298-994e-d472cb075ea7"},"o":{"_id":{"$oid":"618d5a3e4e14855a5232838a"},"processed":0.0,"msg":"did=+xxxxxxxx;voip_id=xxxx;action=test","type":"agent_flagged","created_date":1636653630.0},"ts":{"$timestamp":{"t":1636653630,"i":3}},"t":11,"v":2,"wall":{"$date":"2021-11-11T18:00:30.866Z"},"stmtId":0,"prevOpTime":{"ts":{"$timestamp":{"t":0,"i":0}},"t":-1}},"lastOperation":{"lsid":{"id":{"$uuid":"82e67257-6cc3-4f85-b533-39d6db763148"},"uid":{"$binary":{"base64":"Js8qjzFX7fsgL9xTiVDYTvo/U37kp5k1eA8gdWBPtQg=","subType":"0"}}},"txnNumber":1,"op":"i","ns":"fpid.events","ui":{"$uuid":"587beef0-6621-4298-994e-d472cb075ea7"},"o":{"_id":{"$oid":"618d5a3e4e14855a5232838a"},"processed":0.0,"msg":"did=+xxxxxxxx;voip_id=xxxx;action=test","type":"agent_flagged","created_date":1636653630.0},"ts":{"$timestamp":{"t":1636653630,"i":3}},"t":11,"v":2,"wall":{"$date":"2021-11-11T18:00:30.866Z"},"stmtId":0,"prevOpTime":{"ts":{"$timestamp":{"t":0,"i":0}},"t":-1}},"failedWriterThread":11,"error":"NamespaceNotFound: Failed to apply operation: { lsid: { id: UUID(\"82e67257-6cc3-4f85-b533-39d6db763148\"), uid: BinData(0, 26CF2A8F3157EDFB202FDC538950D84EFA3F537EE4A79935780F2075604FB508) }, txnNumber: 1, op: \"i\", ns: \"fpid.events\", ui: UUID(\"587beef0-6621-4298-994e-d472cb075ea7\"), o: { _id: ObjectId('618d5a3e4e14855a5232838a'), processed: 0.0, msg: \"did=+xxxxxxxx;voip_id=xxxx;action=test\", type: \"agent_flagged\", created_date: 1636653630.0 }, ts: Timestamp(1636653630, 3), t: 11, v: 2, wall: new Date(1636653630866), stmtId: 0, prevOpTime: { ts: Timestamp(0, 0), t: -1 } } :: caused by :: Unable to resolve 587beef0-6621-4298-994e-d472cb075ea7"}}
|
{"t":{"$date":"2021-11-11T10:00:30.868-08:00"},"s":"F", "c":"-", "id":23095, "ctx":"OplogApplier-0","msg":"Fatal assertion","attr":{"msgid":34437,"error":"NamespaceNotFound: Failed to apply operation: { lsid: { id: UUID(\"82e67257-6cc3-4f85-b533-39d6db763148\"), uid: BinData(0, 26CF2A8F3157EDFB202FDC538950D84EFA3F537EE4A79935780F2075604FB508) }, txnNumber: 1, op: \"i\", ns: \"fpid.events\", ui: UUID(\"587beef0-6621-4298-994e-d472cb075ea7\"), o: { _id: ObjectId('618d5a3e4e14855a5232838a'), processed: 0.0, msg: \"did=+xxxxxxxx;voip_id=xxxx;action=test\", type: \"agent_flagged\", created_date: 1636653630.0 }, ts: Timestamp(1636653630, 3), t: 11, v: 2, wall: new Date(1636653630866), stmtId: 0, prevOpTime: { ts: Timestamp(0, 0), t: -1 } } :: caused by :: Unable to resolve 587beef0-6621-4298-994e-d472cb075ea7","file":"src/mongo/db/repl/oplog_applier_impl.cpp","line":510}}
|
{"t":{"$date":"2021-11-11T10:00:30.868-08:00"},"s":"F", "c":"-", "id":23096, "ctx":"OplogApplier-0","msg":"\n\n***aborting after fassert() failure\n\n"}
|
after this error, we are not able to start this MongoDB service on this node, it's failing on restart.
Could you please suggest what could be the reason for this and how we can prevent this error?
|