----- END BACKTRACE ----- 2018-04-27T02:42:33.012+0000 F - [replication-100] terminate() called. An exception is active; attempting to gather more information 2018-04-27T02:42:33.033+0000 F - [replication-100] DBException::toString(): 10334 BSONObj size: 38694152 (0x24E6D08) is invalid. Size must be between 0 and 16793600(16MB) First element: databasesCloned: 56584 Actual exception type: mongo::MsgAssertionException 0x559ea864c641 0x559ea864c025 0x559ea90a6ca6 0x559ea913a979 0x559ea90a6645 0x7fa10ab5fff3 0x7fa10ab60517 0x559ea80259f2 0x559ea8028c45 0x559ea7965826 0x559ea80290ef 0x559ea80681ea 0x559ea8068357 0x559ea806e293 0x559ea79fbe50 0x559ea83dc45a 0x559ea83df853 0x559ea83dfd2b 0x559ea85c654c 0x559ea85c6ffc 0x559ea85c79e6 0x559ea90c1a40 0x7fa10a93b064 0x7fa10a67062d ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"559EA70DB000","o":"1571641","s":"_ZN5mongo15printStackTraceERSo"},{"b":"559EA70DB000","o":"1571025"},{"b":"559EA70DB000","o":"1FCBCA6","s":"_ZN10__cxxabiv111__terminateEPFvvE"},{"b":"559EA70DB000","o":"205F979"},{"b":"559EA70DB000","o":"1FCB645","s":"__gxx_personality_v0"},{"b":"7FA10AB50000","o":"FFF3"},{"b":"7FA10AB50000","o":"10517","s":"_Unwind_Resume"},{"b":"559EA70DB000","o":"F4A9F2","s":"_ZN5mongo4repl13InitialSyncer25_finishInitialSyncAttemptERKNS_10StatusWithINS0_10OpTimeWithIxEEEE"},{"b":"559EA70DB000","o":"F4DC45","s":"_ZN5mongo4repl23CallbackCompletionGuardINS_10StatusWithINS0_10OpTimeWithIxEEEEED1Ev"},{"b":"559EA70DB000","o":"88A826","s":"_ZNSt16_Sp_counted_baseILN9__gnu_cxx12_Lock_policyE2EE10_M_releaseEv"},{"b":"559EA70DB000","o":"F4E0EF","s":"_ZNSt14_Function_base13_Base_managerISt5_BindIFSt7_Mem_fnIMN5mongo4repl13InitialSyncerEFvRKNS3_6StatusERKNS4_10OpTimeWithIxEESt10shared_ptrINS4_23CallbackCompletionGuardINS3_10StatusWithISA_EEEEEEEPS5_St12_PlaceholderILi1EESN_ILi2EESI_EEE10_M_managerERSt9_Any_dataRKST_St18_Manager_operation"},{"b":"559EA70DB000","o":"F8D1EA","s":"_ZN5mongo4repl12OplogFetcher15_finishCallbackENS_6StatusENS0_10OpTimeWithIxEE"},{"b":"559EA70DB000","o":"F8D357","s":"_ZN5mongo4repl12OplogFetcher15_finishCallbackENS_6StatusE"},{"b":"559EA70DB000","o":"F93293","s":"_ZN5mongo4repl12OplogFetcher9_callbackERKNS_10StatusWithINS_7Fetcher13QueryResponseEEEPNS_14BSONObjBuilderE"},{"b":"559EA70DB000","o":"920E50","s":"_ZN5mongo7Fetcher9_callbackERKNS_8executor12TaskExecutor25RemoteCommandCallbackArgsEPKc"},{"b":"559EA70DB000","o":"130145A"},{"b":"559EA70DB000","o":"1304853","s":"_ZN5mongo8executor22ThreadPoolTaskExecutor11runCallbackESt10shared_ptrINS1_13CallbackStateEE"},{"b":"559EA70DB000","o":"1304D2B"},{"b":"559EA70DB000","o":"14EB54C","s":"_ZN5mongo10ThreadPool10_doOneTaskEPSt11unique_lockISt5mutexE"},{"b":"559EA70DB000","o":"14EBFFC","s":"_ZN5mongo10ThreadPool13_consumeTasksEv"},{"b":"559EA70DB000","o":"14EC9E6","s":"_ZN5mongo10ThreadPool17_workerThreadBodyEPS0_RKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE"},{"b":"559EA70DB000","o":"1FE6A40"},{"b":"7FA10A933000","o":"8064"},{"b":"7FA10A588000","o":"E862D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.4.10", "gitVersion" : "078f28920cb24de0dd479b5ea6c66c644f6326e9", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.16.0-4-amd64", "version" : "#1 SMP Debian 3.16.51-2 (2017-12-03)", "machine" : "x86_64" }, "somap" : [ { "b" : "559EA70DB000", "elfType" : 3, "buildId" : "004E9FAB8E542F8BABDB0274A211B1E9D15DEDF1" }, { "b" : "7FFEC2116000", "path" : "linux-vdso.so.1", "elfType" : 3, "buildId" : "E09EDF91CF4D4B631FC9B5CE3519FF61F23C2926" }, { "b" : "7FA10B86F000", "path" : "/usr/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "21115992A1F885E1ACE88AADA60F126AD9759D03" }, { "b" : "7FA10B473000", "path" : "/usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "FD6376149047833953B0269E84DE181CA45DBE90" }, { "b" : "7FA10B26B000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "A63C95FB33CCA970E141D2E13774B997C1CF0565" }, { "b" : "7FA10B067000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "D70B531D672A34D71DB42EB32B68E63F2DCC5B6A" }, { "b" : "7FA10AD66000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "152C93BA3E8590F7ED0BCDDF868600D55EC4DD6F" }, { "b" : "7FA10AB50000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "D5FB04F64B3DAEA6D6B68B5E8B9D4D2BC1A6E1FC" }, { "b" : "7FA10A933000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "9DA9387A60FFC196AEDB9526275552AFEF499C44" }, { "b" : "7FA10A588000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "48C48BC6ABB794461B8A558DD76B29876A0551F0" }, { "b" : "7FA10BAD0000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "1D98D41FBB1EABA7EC05D0FD7624B85D6F51C03C" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x559ea864c641] mongod(+0x1571025) [0x559ea864c025] mongod(_ZN10__cxxabiv111__terminateEPFvvE+0x6) [0x559ea90a6ca6] mongod(+0x205F979) [0x559ea913a979] mongod(__gxx_personality_v0+0x2B5) [0x559ea90a6645] libgcc_s.so.1(+0xFFF3) [0x7fa10ab5fff3] libgcc_s.so.1(_Unwind_Resume+0x57) [0x7fa10ab60517] mongod(_ZN5mongo4repl13InitialSyncer25_finishInitialSyncAttemptERKNS_10StatusWithINS0_10OpTimeWithIxEEEE+0x7B2) [0x559ea80259f2] mongod(_ZN5mongo4repl23CallbackCompletionGuardINS_10StatusWithINS0_10OpTimeWithIxEEEEED1Ev+0x25) [0x559ea8028c45] mongod(_ZNSt16_Sp_counted_baseILN9__gnu_cxx12_Lock_policyE2EE10_M_releaseEv+0x66) [0x559ea7965826] mongod(_ZNSt14_Function_base13_Base_managerISt5_BindIFSt7_Mem_fnIMN5mongo4repl13InitialSyncerEFvRKNS3_6StatusERKNS4_10OpTimeWithIxEESt10shared_ptrINS4_23CallbackCompletionGuardINS3_10StatusWithISA_EEEEEEEPS5_St12_PlaceholderILi1EESN_ILi2EESI_EEE10_M_managerERSt9_Any_dataRKST_St18_Manager_operation+0x3F) [0x559ea80290ef] mongod(_ZN5mongo4repl12OplogFetcher15_finishCallbackENS_6StatusENS0_10OpTimeWithIxEE+0x14A) [0x559ea80681ea] mongod(_ZN5mongo4repl12OplogFetcher15_finishCallbackENS_6StatusE+0x97) [0x559ea8068357] mongod(_ZN5mongo4repl12OplogFetcher9_callbackERKNS_10StatusWithINS_7Fetcher13QueryResponseEEEPNS_14BSONObjBuilderE+0x513) [0x559ea806e293] mongod(_ZN5mongo7Fetcher9_callbackERKNS_8executor12TaskExecutor25RemoteCommandCallbackArgsEPKc+0x22B0) [0x559ea79fbe50] mongod(+0x130145A) [0x559ea83dc45a] mongod(_ZN5mongo8executor22ThreadPoolTaskExecutor11runCallbackESt10shared_ptrINS1_13CallbackStateEE+0x1B3) [0x559ea83df853] mongod(+0x1304D2B) [0x559ea83dfd2b] mongod(_ZN5mongo10ThreadPool10_doOneTaskEPSt11unique_lockISt5mutexE+0x14C) [0x559ea85c654c] mongod(_ZN5mongo10ThreadPool13_consumeTasksEv+0xBC) [0x559ea85c6ffc] mongod(_ZN5mongo10ThreadPool17_workerThreadBodyEPS0_RKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE+0x96) [0x559ea85c79e6] mongod(+0x1FE6A40) [0x559ea90c1a40] libpthread.so.0(+0x8064) [0x7fa10a93b064] libc.so.6(clone+0x6D) [0x7fa10a67062d] ----- END BACKTRACE -- After MongoD Restart after sync crash resulting in dropAllDatabasesExceptLocal Initializing full-time diagnostic data capture with directory '/mnt/nvme/mongodb/diagnostic.data' 2018-04-27T03:05:00.009+0000 I FTDC [ftdc] Unclean full-time diagnostic data capture shutdown detected, found interim file, some metrics may have been lost. OK 2018-04-27T03:05:00.552+0000 I REPL [initandlisten] Did not find local voted for document at startup. 2018-04-27T03:05:00.806+0000 I NETWORK [thread1] waiting for connections on port 27017 2018-04-27T03:05:00.899+0000 W REPL [replExecDBWorker-1] Failed to load timestamp of most recently applied operation: InitialSyncFailure: In the middle of an initial sync. 2018-04-27T03:05:00.905+0000 I REPL [replExecDBWorker-1] New replica set config in use: { _id: "Removed", version: 313676, members: [ { _id: 9, host: "Removed", arbiterOnly: false, buildIndexes: true, hidden: false, priority: 0.5, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 10, host: "Removed", arbiterOnly: true, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 11, host: "Removed", arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 } ], settings: { chainingAllowed: true, heartbeatIntervalMillis: 2000, heartbeatTimeoutSecs: 10, electionTimeoutMillis: 10000, catchUpTimeoutMillis: 60000, getLastErrorModes: {}, getLastErrorDefaults: { w: 1, wtimeout: 0 } } } 2018-04-27T03:05:00.905+0000 I REPL [replExecDBWorker-1] This node is IP_REMOVED:27017 in the config 2018-04-27T03:05:00.905+0000 I REPL [replExecDBWorker-1] transition to STARTUP2 2018-04-27T03:05:00.906+0000 I REPL [replExecDBWorker-1] Starting replication storage threads 2018-04-27T03:05:00.938+0000 I REPL [replication-0] Starting initial sync (attempt 1 of 10) [replication-1] dropAllDatabasesExceptLocal 56592