2017-11-16T05:49:33.133+0100 I REPL [replication-115] Finished fetching oplog during initial sync: CallbackCanceled: Callback cance led. Last fetched optime and hash: { ts: Timestamp 1510807773000|11, t: 11 }[-3242675976573290350] 2017-11-16T05:49:33.268+0100 I CONTROL [replication-115] 0x7fb1aef20bc1 0x7fb1aeeaba17 0x7fb1ae1bb8e5 0x7fb1ae1bb9c4 0x7fb1ae255798 0x7fb1ae245b28 0x7fb1ae24b6c2 0x7fb1ae256292 0x7fb1ae256566 0x7fb1aecbffad 0x7fb1ae8ec8fb 0x7fb1ae8f1ce5 0x7fb1ae229736 0x7fb1ae8f08ff 0x7fb1ae93108a 0x7fb1ae9311f7 0x7fb1ae936fa3 0x7fb1ae2beaf0 0x7fb1aecae27a 0x7fb1aecb1683 0x7fb1aecb1b5b 0x7fb1aee98795 0x7fb1aee992c0 0x7fb1aee99e69 0x7fb1af98f690 0x7fb1ac5f2064 0x7fb1ac32762d ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"7FB1AD9AA000","o":"1576BC1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"7FB1AD9AA000","o":"1501A17","s":"_ZN5mongo10logContextEPKc"},{"b":"7FB1AD9AA000","o":"8118E5","s":"_ZN5mongo23msgassertedWithLocationEiPKcS1_j"},{"b":"7FB1AD9AA000","o":"8119C4","s":"_ZN5mongo30msgassertedNoTraceWithLocationEiPKcS1_j"},{"b":"7FB1AD9AA000","o":"8AB798","s":"_ZNK5mongo7BSONObj14_assertInvalidEv"},{"b":"7FB1AD9AA000","o":"89BB28","s":"_ZNK5mongo11BSONElement14embeddedObjectEv"},{"b":"7FB1AD9AA000","o":"8A16C2","s":"_ZNK5mongo11BSONElement8toStringERNS_17StringBuilderImplINS_21SharedBufferAllocatorEEEbbbi"},{"b":"7FB1AD9AA000","o":"8AC292","s":"_ZNK5mongo7BSONObj8toStringERNS_17StringBuilderImplINS_21SharedBufferAllocatorEEEbbbi"},{"b":"7FB1AD9AA000","o":"8AC566","s":"_ZNK5mongo7BSONObj8toStringB5cxx11Eb"},{"b":"7FB1AD9AA000","o":"1315FAD","s":"_ZN5mongo6redactB5cxx11ERKNS_7BSONObjE"},{"b":"7FB1AD9AA000","o":"F428FB","s":"_ZN5mongo4repl13InitialSyncer25_finishInitialSyncAttemptERKNS_10StatusWithINS0_10OpTimeWithIxEEEE"},{"b":"7FB1AD9AA000","o":"F47CE5","s":"_ZN5mongo4repl23CallbackCompletionGuardINS_10StatusWithINS0_10OpTimeWithIxEEEEED1Ev"},{"b":"7FB1AD9AA000","o":"87F736","s":"_ZNSt16_Sp_counted_baseILN9__gnu_cxx12_Lock_policyE2EE10_M_releaseEv"},{"b":"7FB1AD9AA000","o":"F468FF","s":"_ZNSt14_Function_base13_Base_managerISt5_BindIFSt7_Mem_fnIMN5mongo4repl13InitialSyncerEFvRKNS3_6StatusERKNS4_10OpTimeWithIxEESt10shared_ptrINS4_23CallbackCompletionGuardINS3_10StatusWithISA_EEEEEEEPS5_St12_PlaceholderILi1EESN_ILi2EESI_EEE10_M_managerERSt9_Any_dataRKST_St18_Manager_operation"},{"b":"7FB1AD9AA000","o":"F8708A","s":"_ZN5mongo4repl12OplogFetcher15_finishCallbackENS_6StatusENS0_10OpTimeWithIxEE"},{"b":"7FB1AD9AA000","o":"F871F7","s":"_ZN5mongo4repl12OplogFetcher15_finishCallbackENS_6StatusE"},{"b":"7FB1AD9AA000","o":"F8CFA3","s":"_ZN5mongo4repl12OplogFetcher9_callbackERKNS_10StatusWithINS_7Fetcher13QueryResponseEEEPNS_14BSONObjBuilderE"},{"b":"7FB1AD9AA000","o":"914AF0","s":"_ZN5mongo7Fetcher9_callbackERKNS_8executor12TaskExecutor25RemoteCommandCallbackArgsEPKc"},{"b":"7FB1AD9AA000","o":"130427A"},{"b":"7FB1AD9AA000","o":"1307683","s":"_ZN5mongo8executor22ThreadPoolTaskExecutor11runCallbackESt10shared_ptrINS1_13CallbackStateEE"},{"b":"7FB1AD9AA000","o":"1307B5B"},{"b":"7FB1AD9AA000","o":"14EE795","s":"_ZN5mongo10ThreadPool10_doOneTaskEPSt11unique_lockISt5mutexE"},{"b":"7FB1AD9AA000","o":"14EF2C0","s":"_ZN5mongo10ThreadPool13_consumeTasksEv"},{"b":"7FB1AD9AA000","o":"14EFE69","s":"_ZN5mongo10ThreadPool17_workerThreadBodyEPS0_RKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE"},{"b":"7FB1AD9AA000","o":"1FE5690"},{"b":"7FB1AC5EA000","o":"8064"},{"b":"7FB1_RKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE"},{"b":"7FB1AD9AA000","o":"1FE5690"},{"b":"7FB1AC5EA000","o":"8064"},{"b":"7FB1 AC23F000","o":"E862D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.4.4", "gitVersion" : "888390515874a9debd1b6c5d36559ca86b44babd", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.16.0-4-amd64", "version" : "#1 SMP Debian 3.16.39-1 (2016-12-30)", "machine" : "x86_64" }, "somap" : [ { "b" : "7FB1AD9AA000", "elfType" : 3, "buildId" : "59518566540C4D32B0F28CDEADF506ADAECCE6D1" }, { "b" : "7FFE53A3D000", "path" : "linux-vdso.so.1", "elfType" : 3, "buildId" : "856A20F5A861E2D4F656A3D865B4D6158E2D607F" }, { "b" : "7FB1AD526000", "path" : "/usr/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "0711D67B299ACA3EA6B0B57C0DA4EC83B71787EE" }, { "b" : "7FB1AD12A000", "path" : "/usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "2E613A282B22F2B0978ACCEC1B71411F5E20CC89" }, { "b" : "7FB1ACF22000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "A63C95FB33CCA970E141D2E13774B997C1CF0565" }, { "b" : "7FB1ACD1E000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "D70B531D672A34D71DB42EB32B68E63F2DCC5B6A" }, { "b" : "7FB1ACA1D000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "152C93BA3E8590F7ED0BCDDF868600D55EC4DD6F" }, { "b" : "7FB1AC807000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "D5FB04F64B3DAEA6D6B68B5E8B9D4D2BC1A6E1FC" }, { "b" : "7FB1AC5EA000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "9DA9387A60FFC196AEDB9526275552AFEF499C44" }, { "b" : "7FB1AC23F000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "48C48BC6ABB794461B8A558DD76B29876A0551F0" }, { "b" : "7FB1AD787000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "1D98D41FBB1EABA7EC05D0FD7624B85D6F51C03C" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x7fb1aef20bc1] mongod(_ZN5mongo10logContextEPKc+0x177) [0x7fb1aeeaba17] mongod(_ZN5mongo23msgassertedWithLocationEiPKcS1_j+0x1C9) [0x7fb1ae1bb8e5] mongod(_ZN5mongo30msgassertedNoTraceWithLocationEiPKcS1_j+0x0) [0x7fb1ae1bb9c4] mongod(_ZNK5mongo7BSONObj14_assertInvalidEv+0x208) [0x7fb1ae255798] mongod(_ZNK5mongo11BSONElement14embeddedObjectEv+0x78) [0x7fb1ae245b28] mongod(_ZNK5mongo11BSONElement8toStringERNS_17StringBuilderImplINS_21SharedBufferAllocatorEEEbbbi+0x2D2) [0x7fb1ae24b6c2] mongod(_ZNK5mongo7BSONObj8toStringERNS_17StringBuilderImplINS_21SharedBufferAllocatorEEEbbbi+0x202) [0x7fb1ae256292] mongod(_ZNK5mongo7BSONObj8toStringB5cxx11Eb+0x96) [0x7fb1ae256566] mongod(_ZN5mongo6redactB5cxx11ERKNS_7BSONObjE+0x6D) [0x7fb1aecbffad] mongod(_ZN5mongo4repl13InitialSyncer25_finishInitialSyncAttemptERKNS_10StatusWithINS0_10OpTimeWithIxEEEE+0x10B) [0x7fb1ae8ec8fb] mongod(_ZN5mongo4repl23CallbackCompletionGuardINS_10StatusWithINS0_10OpTimeWithIxEEEEED1Ev+0x25) [0x7fb1ae8f1ce5] mongod(_ZNSt16_Sp_counted_baseILN9__gnu_cxx12_Lock_policyE2EE10_M_releaseEv+0x66) [0x7fb1ae229736] mongod(_ZNSt14_Function_base13_Base_managerISt5_BindIFSt7_Mem_fnIMN5mongo4repl13InitialSyncerEFvRKNS3_6StatusERKNS4_10OpTimeWithIxEESt10shared_ptrINS4_23CallbackCompletionGuardINS3_10StatusWithISA_EEEEEEEPS5_St12_PlaceholderILi1EESN_ILi2EESI_EEE10_M_managerERSt9_Any_dataRKST_St18_Manager_operation+0x3F) [0x7fb1ae8f08ff] mongod(_ZN5mongo4repl12OplogFetcher15_finishCallbackENS_6StatusENS0_10OpTimeWithIxEE+0x14A) [0x7fb1ae93108a] mongod(_ZN5mongo4repl12OplogFetcher15_finishCallbackENS_6StatusE+0x97) [0x7fb1ae9311f7] mongod(_ZN5mongo4repl12OplogFetcher9_callbackERKNS_10StatusWithINS_7Fetcher13QueryResponseEEEPNS_14BSONObjBuilderE+0x553) [0x7fb1ae936fa3] mongod(_ZN5mongo7Fetcher9_callbackERKNS_8executor12TaskExecutor25RemoteCommandCallbackArgsEPKc+0x22B0) [0x7fb1ae2beaf0] mongod(+0x130427A) [0x7fb1aecae27a] mongod(_ZN5mongo8executor22ThreadPoolTaskExecutor11runCallbackESt10shared_ptrINS1_13CallbackStateEE+0x1B3) [0x7fb1aecb1683] mongod(+0x1307B5B) [0x7fb1aecb1b5b] mongod(_ZN5mongo10ThreadPool10_doOneTaskEPSt11unique_lockISt5mutexE+0x135) [0x7fb1aee98795] mongod(_ZN5mongo10ThreadPool13_consumeTasksEv+0xC0) [0x7fb1aee992c0] mongod(_ZN5mongo10ThreadPool17_workerThreadBodyEPS0_RKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE+0x149) [0x7fb1aee99e69] mongod(+0x1FE5690) [0x7fb1af98f690] libpthread.so.0(+0x8064) [0x7fb1ac5f2064] libc.so.6(clone+0x6D) [0x7fb1ac32762d] ----- END BACKTRACE ----- 2017-11-16T05:49:33.270+0100 F - [replication-115] terminate() called. An exception is active; attempting to gather more information 2017-11-16T05:49:33.282+0100 F - [replication-115] DBException::toString(): 10334 BSONObj size: 32985739 (0x1F7528B) is invalid. Size must be between 0 and 16793600(16MB) First element: databasesCloned: 10191 Actual exception type: mongo::MsgAssertionException 0x7fb1aef20bc1 0x7fb1aef20485 0x7fb1af9748f6 0x7fb1afa085c9 0x7fb1af974295 0x7fb1ac816ff3 0x7fb1ac817517 0x7fb1ae8ecf42 0x7fb1ae8f1ce5 0x7fb1ae229736 0x7fb1ae8f08ff 0x7fb1ae93108a 0x7fb1ae9311f7 0x7fb1ae936fa3 0x7fb1ae2beaf0 0x7fb1aecae27a 0x7fb1aecb1683 0x7fb1aecb1b5b 0x7fb1aee98795 0x7fb1aee992c0 0x7fb1aee99e69 0x7fb1af98f690 0x7fb1ac5f2064 0x7fb1ac32762d ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"7FB1AD9AA000","o":"1576BC1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"7FB1AD9AA000","o":"1576485"},{"b":"7FB1AD9AA000","o":"1FCA8F6","s":"_ZN10__cxxabiv111__terminateEPFvvE"},{"b":"7FB1AD9AA000","o":"205E5C9"},{"b":"7FB1AD9AA000","o":"1FCA295","s":"__gxx_personality_v0"},{"b":"7FB1AC807000","o":"FFF3"},{"b":"7FB1AC807000","o":"10517","s":"_Unwind_Resume"},{"b":"7FB1AD9AA000","o":"F42F42","s":"_ZN5mongo4repl13InitialSyncer25_finishInitialSyncAttemptERKNS_10StatusWithINS0_10OpTimeWithIxEEEE"},{"b":"7FB1AD9AA000","o":"F47CE5","s":"_ZN5mongo4repl23CallbackCompletionGuardINS_10StatusWithINS0_10OpTimeWithIxEEEEED1Ev"},{"b":"7FB1AD9AA000","o":"87F736","s":"_ZNSt16_Sp_counted_baseILN9__gnu_cxx12_Lock_policyE2EE10_M_releaseEv"},{"b":"7FB1AD9AA000","o":"F468FF","s":"_ZNSt14_Function_base13_Base_managerISt5_BindIFSt7_Mem_fnIMN5mongo4repl13InitialSyncerEFvRKNS3_6StatusERKNS4_10OpTimeWithIxEESt10shared_ptrINS4_23CallbackCompletionGuardINS3_10StatusWithISA_EEEEEEEPS5_St12_PlaceholderILi1EESN_ILi2EESI_EEE10_M_managerERSt9_Any_dataRKST_St18_Manager_operation"},{"b":"7FB1AD9AA000","o":"F8708A","s":"_ZN5mongo4repl12OplogFetcher15_finishCallbackENS_6StatusENS0_10OpTimeWithIxEE"},{"b":"7FB1AD9AA000","o":"F871F7","s":"_ZN5mongo4repl12OplogFetcher15_finishCallbackENS_6StatusE"},{"b":"7FB1AD9AA000","o":"F8CFA3","s":"_ZN5mongo4repl12OplogFetcher9_callbackERKNS_10StatusWithINS_7Fetcher13QueryResponseEEEPNS_14BSONObjBuilderE"},{"b":"7FB1AD9AA000","o":"914AF0","s":"_ZN5mongo7Fetcher9_callbackERKNS_8executor12TaskExecutor25RemoteCommandCallbackArgsEPKc"},{"b":"7FB1AD9AA000","o":"130427A"},{"b":"7FB1AD9AA000","o":"1307683","s":"_ZN5mongo8executor22ThreadPoolTaskExecutor11runCallbackESt10shared_ptrINS1_13CallbackStateEE"},{"b":"7FB1AD9AA000","o":"1307B5B"},{"b":"7FB1AD9AA000","o":"14EE795","s":"_ZN5mongo10ThreadPool10_doOneTaskEPSt11unique_lockISt5mutexE"},{"b":"7FB1AD9AA000","o":"14EF2C0","s":"_ZN5mongo10ThreadPool13_consumeTasksEv"},{"b":"7FB1AD9AA000","o":"14EFE69","s":"_ZN5mongo10ThreadPool17_workerThreadBodyEPS0_RKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE"},{"b":"7FB1AD9AA000","o":"1FE5690"},{"b":"7FB1AC5EA000","o":"8064"},{"b":"7FB1AC23F000","o":"E862D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.4.4", "gitVersion" : "888390515874a9debd1b6c5d36559ca86b44babd", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.16.0-4-amd64", "version" : "#1 SMP Debian 3.16.39-1 (2016-12-30)", "machine" : "x86_64" }, "somap" : [ { "b" : "7FB1AD9AA000", "elfType" : 3, "buildId" : "59518566540C4D32B0F28CDEADF506ADAECCE6D1" }, { "b" : "7FFE53A3D000", "path" : "linux-vdso.so.1", "elfType" : 3, "buildId" : "856A20F5A861E2D4F656A3D865B4D6158E2D607F" }, { "b" : "7FB1AD526000", "path" : "/usr/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "0711D67B299ACA3EA6B0B57C0DA4EC83B71787EE" }, { "b" : "7FB1AD12A000", "path" : "/usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "2E613A282B22F2B0978ACCEC1B71411F5E20CC89" }, { "b" : "7FB1ACF22000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "A63C95FB33CCA970E141D2E13774B997C1CF0565" }, { "b" : "7FB1ACD1E000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "D70B531D672A34D71DB42EB32B68E63F2DCC5B6A" }, { "b" : "7FB1ACA1D000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "152C93BA3E8590F7ED0BCDDF868600D55EC4DD6F" }, { "b" : "7FB1AC807000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "D5FB04F64B3DAEA6D6B68B5E8B9D4D2BC1A6E1FC" }, { "b" : "7FB1AC5EA000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "9DA9387A60FFC196AEDB9526275552AFEF499C44" }, { "b" : "7FB1AC23F000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "48C48BC6ABB794461B8A558DD76B29876A0551F0" }, { "b" : "7FB1AD787000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "1D98D41FBB1EABA7EC05D0FD7624B85D6F51C03C" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x7fb1aef20bc1] mongod(+0x1576485) [0x7fb1aef20485] mongod(_ZN10__cxxabiv111__terminateEPFvvE+0x6) [0x7fb1af9748f6] mongod(+0x205E5C9) [0x7fb1afa085c9] mongod(__gxx_personality_v0+0x2B5) [0x7fb1af974295] libgcc_s.so.1(+0xFFF3) [0x7fb1ac816ff3] libgcc_s.so.1(_Unwind_Resume+0x57) [0x7fb1ac817517] mongod(_ZN5mongo4repl13InitialSyncer25_finishInitialSyncAttemptERKNS_10StatusWithINS0_10OpTimeWithIxEEEE+0x752) [0x7fb1ae8ecf42] mongod(_ZN5mongo4repl23CallbackCompletionGuardINS_10StatusWithINS0_10OpTimeWithIxEEEEED1Ev+0x25) [0x7fb1ae8f1ce5] mongod(_ZNSt16_Sp_counted_baseILN9__gnu_cxx12_Lock_policyE2EE10_M_releaseEv+0x66) [0x7fb1ae229736] mongod(_ZNSt14_Function_base13_Base_managerISt5_BindIFSt7_Mem_fnIMN5mongo4repl13InitialSyncerEFvRKNS3_6StatusERKNS4_10OpTimeWithIxEESt10shared_ptrINS4_23CallbackCompletionGuardINS3_10StatusWithISA_EEEEEEEPS5_St12_PlaceholderILi1EESN_ILi2EESI_EEE10_M_managerERSt9_Any_dataRKST_St18_Manager_operation+0x3F) [0x7fb1ae8f08ff] mongod(_ZN5mongo4repl12OplogFetcher15_finishCallbackENS_6StatusENS0_10OpTimeWithIxEE+0x14A) [0x7fb1ae93108a] mongod(_ZN5mongo4repl12OplogFetcher15_finishCallbackENS_6StatusE+0x97) [0x7fb1ae9311f7] mongod(_ZN5mongo4repl12OplogFetcher9_callbackERKNS_10StatusWithINS_7Fetcher13QueryResponseEEEPNS_14BSONObjBuilderE+0x553) [0x7fb1ae936fa3] mongod(_ZN5mongo7Fetcher9_callbackERKNS_8executor12TaskExecutor25RemoteCommandCallbackArgsEPKc+0x22B0) [0x7fb1ae2beaf0] mongod(+0x130427A) [0x7fb1aecae27a] mongod(_ZN5mongo8executor22ThreadPoolTaskExecutor11runCallbackESt10shared_ptrINS1_13CallbackStateEE+0x1B3) [0x7fb1aecb1683] mongod(+0x1307B5B) [0x7fb1aecb1b5b] mongod(_ZN5mongo10ThreadPool10_doOneTaskEPSt11unique_lockISt5mutexE+0x135) [0x7fb1aee98795] mongod(_ZN5mongo10ThreadPool13_consumeTasksEv+0xC0) [0x7fb1aee992c0] mongod(_ZN5mongo10ThreadPool17_workerThreadBodyEPS0_RKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE+0x149) [0x7fb1aee99e69] mongod(+0x1FE5690) [0x7fb1af98f690] libpthread.so.0(+0x8064) [0x7fb1ac5f2064] libc.so.6(clone+0x6D) [0x7fb1ac32762d] ----- END BACKTRACE ----- 2017-11-16T05:57:31.674+0100 I CONTROL [main] ***** SERVER RESTARTED *****