|
When apply a DDL operation like drop/dropIndex/dropDatabase on secondary, if there is a background index building running, the secondary will block all operations(read, write, serverStatus).
background index building
Thread 70 (Thread 0x7f6a3eade700 (LWP 89525)):
|
#0 0x00007f6a5a5aaa5d in __wt_btcur_reset ()
|
#1 0x00007f6a5a5538a9 in ?? ()
|
#2 0x00007f6a5a471952 in mongo::WiredTigerCursor::reset() ()
|
#3 0x00007f6a5a45f83d in mongo::WiredTigerRecordStoreCursorBase::save() ()
|
#4 0x00007f6a5ac212b9 in mongo::PlanExecutor::saveState() ()
|
#5 0x00007f6a5aac724c in mongo::MultiIndexBlockImpl::insertAllDocumentsInCollection(std::set<mongo::RecordId, std::less<mongo::RecordId>, std::allocator<mongo::RecordId> >*) ()
|
#6 0x00007f6a5ab226ab in mongo::IndexBuilder::_build(mongo::OperationContext*, mongo::Database*, bool, mongo::Lock::DBLock*) const ()
|
#7 0x00007f6a5ab23bfa in mongo::IndexBuilder::run() ()
|
#8 0x00007f6a5b96c811 in mongo::BackgroundJob::jobBody() ()
|
#9 0x00007f6a5bfda350 in ?? ()
|
#10 0x00007f6a57e7baa1 in start_thread () from /lib64/libpthread.so.0
|
dropIndex is pending, awaitNoBgOpInProgForNs (this operation call TempRelease to release the lock)
Thread 41 (Thread 0x7f6a39ad6700 (LWP 70411)):
|
#0 0x00007f6a57e7f68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
|
#1 0x00007f6a5bf6f54c in std::condition_variable::wait(std::unique_lock<std::mutex>&) ()
|
#2 0x00007f6a5b2478c3 in ?? ()
|
#3 0x00007f6a5b247b2a in mongo::BackgroundOperation::awaitNoBgOpInProgForNs(mongo::StringData) ()
|
#4 0x00007f6a5ab02669 in mongo::repl::applyCommand_inlock(mongo::OperationContext*, mongo::BSONObj const&, mongo::repl::OplogApplication::Mode) ()
|
#5 0x00007f6a5a74462a in ?? ()
|
#6 0x00007f6a5a748fbe in mongo::repl::SyncTail::syncApply(mongo::OperationContext*, mongo::BSONObj const&, mongo::repl::OplogApplication::Mode) ()
|
#7 0x00007f6a5a74afbc in mongo::repl::multiSyncApply(mongo::OperationContext*, std::vector<mongo::repl::OplogEntry const*, std::allocator<mongo::repl::OplogEntry const*> >*, mongo::repl::SyncTail*, std::vector<mongo::MultikeyPathInfo, std::allocator<mongo::MultikeyPathInfo> >*) ()#8 0x00007f6a5a732288 in std::_Function_handler<mongo::Status (mongo::OperationContext*, std::vector<mongo::repl::OplogEntry const*, std::allocator<mongo::repl::OplogEntry const*> >*, mongo::repl::SyncTail*, std::vector<mongo::MultikeyPathInfo, std::allocator<mongo::MultikeyPathInfo> >*), mongo::Status (*)(mongo::OperationContext*, std::vector<mongo::repl::OplogEntry const*, std::allocator<mongo::repl::OplogEntry const*> >*, mongo::repl::SyncTail*, std::vector<mongo::MultikeyPathInfo, std::allocator<mongo::MultikeyPathInfo> >*)>::_M_invoke(std::_Any_data const&, mongo::OperationContext*&&, std::vector<mongo::repl::OplogEntry const*, std::allocator<mongo::repl::OplogEntry const*> >*&&, mongo::repl::SyncTail*&&, std::vector<mongo::MultikeyPathInfo, std::allocator<mongo::MultikeyPathInfo> >*&&) ()
|
#9 0x00007f6a5a7452f5 in ?? ()
|
#10 0x00007f6a5b5f012c in mongo::ThreadPool::_doOneTask(std::unique_lock<std::mutex>*) ()
|
#11 0x00007f6a5b5f062c in mongo::ThreadPool::_consumeTasks() ()
|
#12 0x00007f6a5b5f1016 in mongo::ThreadPool::_workerThreadBody(mongo::ThreadPool*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) ()
|
#13 0x00007f6a5bfda350 in ?? ()
|
#14 0x00007f6a57e7baa1 in start_thread () from /lib64/libpthread.so.0
|
#15 0x00007f6a57bc893d in clone () from /lib64/libc.so.6
|
serverStatus, and other request is blocking, the secondary will be out of service during the whole index building process.
Thread 71 (Thread 0x7f6a3f5e0700 (LWP 89728)):
|
#0 0x00007f6a57e7fa5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
|
#1 0x00007f6a5b57ae78 in mongo::CondVarLockGrantNotification::wait(mongo::Duration<std::ratio<1l, 1000l> >) ()
|
#2 0x00007f6a5b5819cd in mongo::LockerImpl<false>::lockComplete(mongo::OperationContext*, mongo::ResourceId, mongo::LockMode, mongo::Date_t, bool) ()
|
#3 0x00007f6a5b56e3c1 in mongo::Lock::ResourceLock::lock(mongo::LockMode) ()
|
#4 0x00007f6a5b56e491 in mongo::Lock::GlobalLock::_enqueue(mongo::LockMode, mongo::Date_t) ()
|
#5 0x00007f6a5b56e552 in mongo::Lock::GlobalLock::GlobalLock(mongo::OperationContext*, mongo::LockMode, mongo::Date_t, mongo::Lock::InterruptBehavior, mongo::Lock::GlobalLock::EnqueueOnly) ()
|
#6 0x00007f6a5b56e598 in mongo::Lock::GlobalLock::GlobalLock(mongo::OperationContext*, mongo::LockMode, mongo::Date_t, mongo::Lock::InterruptBehavior) ()
|
#7 0x00007f6a5a440bf8 in mongo::WiredTigerServerStatusSection::generateSection(mongo::OperationContext*, mongo::BSONElement const&) const ()
|
#8 0x00007f6a5b7b4fea in mongo::CmdServerStatus::run(mongo::OperationContext*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, mongo::BSONObj const&, mongo::BSONObjBuilder&) ()
|
#9 0x00007f6a5b85fdb9 in mongo::BasicCommand::Invocation::run(mongo::OperationContext*, mongo::CommandReplyBuilder*) ()
|
#10 0x00007f6a5a42c79f in ?? ()
|
#11 0x00007f6a5a42cbef in ?? ()
|
#12 0x00007f6a5a430df3 in ?? ()
|
#13 0x00007f6a5a432c8f in ?? ()
|
#14 0x00007f6a5a433f9d in mongo::ServiceEntryPointCommon::handleRequest(mongo::OperationContext*, mongo::Message const&, mongo::ServiceEntryPointCommon::Hooks const&) ()
|
#15 0x00007f6a5a41e58a in mongo::ServiceEntryPointMongod::handleRequest(mongo::OperationContext*, mongo::Message const&) ()
|
#16 0x00007f6a5a42a0ca in mongo::ServiceStateMachine::_processMessage(mongo::ServiceStateMachine::ThreadGuard) ()
|
#17 0x00007f6a5a424d87 in mongo::ServiceStateMachine::_runNextInGuard(mongo::ServiceStateMachine::ThreadGuard) ()
|
#18 0x00007f6a5a428591 in ?? ()
|
#19 0x00007f6a5b6ab222 in mongo::transport::ServiceExecutorSynchronous::schedule(std::function<void ()>, mongo::transport::ServiceExecutor::ScheduleFlags, mongo::transport::ServiceExecutorTaskName) ()
|
#20 0x00007f6a5a422faf in mongo::ServiceStateMachine::_scheduleNextWithGuard(mongo::ServiceStateMachine::ThreadGuard, mongo::transport::ServiceExecutor::ScheduleFlags, mongo::transport::ServiceExecutorTaskName, mongo::ServiceStateMachine::Ownership) ()
|
#21 0x00007f6a5a426135 in mongo::ServiceStateMachine::_sourceCallback(mongo::Status) ()
|
#22 0x00007f6a5a4244c7 in mongo::ServiceStateMachine::_sourceMessage(mongo::ServiceStateMachine::ThreadGuard) ()
|
#23 0x00007f6a5a424e0d in mongo::ServiceStateMachine::_runNextInGuard(mongo::ServiceStateMachine::ThreadGuard) ()
|
|