When the server has external (LDAP) authorization configured, a single slow LDAP query will stall all new authorizations, including ones for local users (SCRAM).

4.2.3:

• Long-running LDAP query:

  Thread 65 (Thread 0x7fc7f44ea700 (LWP 23146)):
  #0  0x00007fc809682c2d in poll () from /lib64/libc.so.6
  #1  0x00007fc80cbb37d5 in ldap_result () from /usr/lib64/libldap_r-2.4.so.2
  #2  0x00007fc80cbbc808 in ldap_sasl_bind_s () from /usr/lib64/libldap_r-2.4.so.2
  #3  0x0000564a0797b827 in mongo::(anonymous namespace)::openLDAPBindFunction(ldap*, char const*, unsigned long, int, void*) [clone
   .cold.350] ()
  #4  0x0000564a07ec2162 in mongo::OpenLDAPConnection::bindAsUser(mongo::LDAPBindOptions const&) ()
  #5  0x0000564a07eb4711 in mongo::(anonymous namespace)::WrappedConnection::bindAsUser(mongo::LDAPBindOptions const&) ()
  #6  0x0000564a07eb1f46 in mongo::LDAPRunnerImpl::getConnection() ()
  #7  0x0000564a07eb2b83 in mongo::LDAPRunnerImpl::runQuery[abi:cxx11](mongo::LDAPQuery const&) ()
  #8  0x0000564a07eaced3 in mongo::LDAPManagerImpl::_getGroupDNsFromServer[abi:cxx11](mongo::LDAPQuery&) ()
  #9  0x0000564a079790d8 in mongo::LDAPManagerImpl::getUserRoles(mongo::UserName const&) [clone .cold.235] ()
  #10 0x0000564a07ea6c02 in mongo::AuthzManagerExternalStateLDAP::getUserDescription(mongo::OperationContext*, mongo::UserName const
  &, mongo::BSONObj*) ()
  #11 0x0000564a09047ae5 in mongo::AuthorizationManagerImpl::getUserDescription(mongo::OperationContext*, mongo::UserName const&, mo
  ngo::BSONObj*) ()
  #12 0x0000564a09049f21 in mongo::AuthorizationManagerImpl::_fetchUserV2(mongo::OperationContext*, mongo::UserName const&, std::uni
  que_ptr<mongo::User, std::default_delete<mongo::User> >*) ()
  #13 0x0000564a0904db1b in mongo::AuthorizationManagerImpl::acquireUser(mongo::OperationContext*, mongo::UserName const&) ()
  #14 0x0000564a0904aa4d in mongo::AuthorizationManagerImpl::acquireUserForSessionRefresh(mongo::OperationContext*, mongo::UserName
  const&, std::vector<unsigned char, std::allocator<unsigned char> > const&) ()
  #15 0x0000564a0905c586 in mongo::AuthorizationSessionImpl::_refreshUserInfoAsNeeded(mongo::OperationContext*) ()
  #16 0x0000564a0811b75e in mongo::ServiceEntryPointCommon::handleRequest(mongo::OperationContext*, mongo::Message const&, mongo::Se
  rviceEntryPointCommon::Hooks const&) ()
  #17 0x0000564a08108bdc in mongo::ServiceEntryPointMongod::handleRequest(mongo::OperationContext*, mongo::Message const&) ()
  #18 0x0000564a0811512c in mongo::ServiceStateMachine::_processMessage(mongo::ServiceStateMachine::ThreadGuard) ()
  #19 0x0000564a0811095f in mongo::ServiceStateMachine::_runNextInGuard(mongo::ServiceStateMachine::ThreadGuard) ()
  #20 0x0000564a08113bec in std::_Function_handler<void (), mongo::ServiceStateMachine::_scheduleNextWithGuard(mongo::ServiceStateMa
  chine::ThreadGuard, mongo::transport::ServiceExecutor::ScheduleFlags, mongo::transport::ServiceExecutorTaskName, mongo::ServiceSta
  teMachine::Ownership)::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
  #21 0x0000564a08f1c5a2 in mongo::transport::ServiceExecutorSynchronous::schedule(std::function<void ()>, mongo::transport::Service
  Executor::ScheduleFlags, mongo::transport::ServiceExecutorTaskName) ()
  #22 0x0000564a0810e35d in mongo::ServiceStateMachine::_scheduleNextWithGuard(mongo::ServiceStateMachine::ThreadGuard, mongo::trans
  port::ServiceExecutor::ScheduleFlags, mongo::transport::ServiceExecutorTaskName, mongo::ServiceStateMachine::Ownership) ()
  #23 0x0000564a08111613 in mongo::ServiceStateMachine::_sourceCallback(mongo::Status) ()
  #24 0x0000564a0810fa27 in mongo::ServiceStateMachine::_sourceMessage(mongo::ServiceStateMachine::ThreadGuard) ()
  #25 0x0000564a081108bb in mongo::ServiceStateMachine::_runNextInGuard(mongo::ServiceStateMachine::ThreadGuard) ()
  #26 0x0000564a08113bec in std::_Function_handler<void (), mongo::ServiceStateMachine::_scheduleNextWithGuard(mongo::ServiceStateMa
  chine::ThreadGuard, mongo::transport::ServiceExecutor::ScheduleFlags, mongo::transport::ServiceExecutorTaskName, mongo::ServiceSta
  teMachine::Ownership)::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
  #27 0x0000564a08f1ca0b in std::_Function_handler<void (), mongo::transport::ServiceExecutorSynchronous::schedule(std::function<voi
  d ()>, mongo::transport::ServiceExecutor::ScheduleFlags, mongo::transport::ServiceExecutorTaskName)::{lambda()#1}>::_M_invoke(std:
  :_Any_data const&) ()
  #28 0x0000564a09588ce4 in mongo::(anonymous namespace)::runFunc(void*) ()
  #29 0x00007fc809964ea5 in start_thread () from /lib64/libpthread.so.0
  #30 0x00007fc80968d8cd in clone () from /lib64/libc.so.6
  

• Stuck threads waiting for the lock:

  Thread 59 (Thread 0x7fc7ed8d9700 (LWP 23312)):
  #0  0x00007fc809968a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
  #1  0x0000564a0992965c in std::condition_variable::wait(std::unique_lock<std::mutex>&) ()
  #2  0x0000564a0904d7ac in mongo::AuthorizationManagerImpl::acquireUser(mongo::OperationContext*, mongo::UserName const&) ()
  #3  0x0000564a0904aa4d in mongo::AuthorizationManagerImpl::acquireUserForSessionRefresh(mongo::OperationContext*, mongo::UserName const&, std::vector<unsigned char, std::allocator<unsigned char> > const&) ()
  #4  0x0000564a0905c586 in mongo::AuthorizationSessionImpl::_refreshUserInfoAsNeeded(mongo::OperationContext*) ()
  #5  0x0000564a0811b75e in mongo::ServiceEntryPointCommon::handleRequest(mongo::OperationContext*, mongo::Message const&, mongo::ServiceEntryPointCommon::Hooks const&) ()
  #6  0x0000564a08108bdc in mongo::ServiceEntryPointMongod::handleRequest(mongo::OperationContext*, mongo::Message const&) ()
  #7  0x0000564a0811512c in mongo::ServiceStateMachine::_processMessage(mongo::ServiceStateMachine::ThreadGuard) ()
  #8  0x0000564a0811095f in mongo::ServiceStateMachine::_runNextInGuard(mongo::ServiceStateMachine::ThreadGuard) ()
  #9  0x0000564a08113bec in std::_Function_handler<void (), mongo::ServiceStateMachine::_scheduleNextWithGuard(mongo::ServiceStateMachine::ThreadGuard, mongo::transport::ServiceExecutor::ScheduleFlags, mongo::transport::ServiceExecutorTaskName, mongo::ServiceStateMachine::Ownership)::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
  #10 0x0000564a08f1c5a2 in mongo::transport::ServiceExecutorSynchronous::schedule(std::function<void ()>, mongo::transport::ServiceExecutor::ScheduleFlags, mongo::transport::ServiceExecutorTaskName) ()
  #11 0x0000564a0810e35d in mongo::ServiceStateMachine::_scheduleNextWithGuard(mongo::ServiceStateMachine::ThreadGuard, mongo::transport::ServiceExecutor::ScheduleFlags, mongo::transport::ServiceExecutorTaskName, mongo::ServiceStateMachine::Ownership) ()
  #12 0x0000564a08111613 in mongo::ServiceStateMachine::_sourceCallback(mongo::Status) ()
  #13 0x0000564a0810fa27 in mongo::ServiceStateMachine::_sourceMessage(mongo::ServiceStateMachine::ThreadGuard) ()
  #14 0x0000564a081108bb in mongo::ServiceStateMachine::_runNextInGuard(mongo::ServiceStateMachine::ThreadGuard) ()
  #15 0x0000564a08113bec in std::_Function_handler<void (), mongo::ServiceStateMachine::_scheduleNextWithGuard(mongo::ServiceStateMachine::ThreadGuard, mongo::transport::ServiceExecutor::ScheduleFlags, mongo::transport::ServiceExecutorTaskName, mongo::ServiceStateMachine::Ownership)::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
  #16 0x0000564a08f1ca0b in std::_Function_handler<void (), mongo::transport::ServiceExecutorSynchronous::schedule(std::function<void ()>, mongo::transport::ServiceExecutor::ScheduleFlags, mongo::transport::ServiceExecutorTaskName)::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
  #17 0x0000564a09588ce4 in mongo::(anonymous namespace)::runFunc(void*) ()
  #18 0x00007fc809964ea5 in start_thread () from /lib64/libpthread.so.0
  #19 0x00007fc80968d8cd in clone () from /lib64/libc.so.6
  

Since authorization cache flushes occur every 30 seconds in a default configuration, every long-running connection which previously authenticated using the PLAIN authentication mechanism will be reaching out to the LDAP server during the next operation, queing up behind each other.