mongodb-waitsfor-graph # Legend: # Thread 1 -> Lock 1 indicates Thread 1 is waiting on Lock 1 # Lock 2 -> Thread 2 indicates Lock 2 is held by Thread 2 # No cycle detected in the graph digraph "mongod+lock-status" { "Lock 0x55b6304c0008" -> "Thread 0x7fac2b0ff700"; "Thread 0x7fac2a8fe700" -> "Lock 0x55b630d55c00"; "Thread 0x7fac48f8b700" -> "Lock 0x55b630d55c00"; "Lock 0x55b630d55c00" -> "Thread 0x7fac2b0ff700"; "Thread 0x7fac49f8d700" -> "Lock 0x55b6304c0008"; "Thread 0x7fac44782700" -> "Lock 0x55b6304c0008"; "Thread 0x7fac4978c700" -> "Lock 0x55b6304c0008"; "Thread 0x7fac4b790700" -> "Lock 0x55b630d55c00"; "Lock 0x55b63002b618" -> "Thread 0x7fac5179c700"; "Thread 0x7fac2b0ff700" -> "Lock 0x55b63002b618"; "Thread 0x7fac46473700" -> "Lock 0x55b6304c0008"; "Thread 0x7fac5179c700" [label="Thread 0x7fac5179c700 (LWP 32228)" ] "Lock 0x55b6304c0008" [label="Lock 0x55b6304c0008 (Mutex)" ] "Thread 0x7fac2a8fe700" [label="Thread 0x7fac2a8fe700 (LWP 1262)" ] "Thread 0x7fac48f8b700" [label="Thread 0x7fac48f8b700 (LWP 32342)" ] "Lock 0x55b630d55c00" [label="Lock 0x55b630d55c00 (MongoDB lock)" ] "Thread 0x7fac49f8d700" [label="Thread 0x7fac49f8d700 (LWP 907)" ] "Thread 0x7fac44782700" [label="Thread 0x7fac44782700 (LWP 32323)" ] "Thread 0x7fac4978c700" [label="Thread 0x7fac4978c700 (LWP 32341)" ] "Thread 0x7fac4b790700" [label="Thread 0x7fac4b790700 (LWP 32337)" ] "Lock 0x55b63002b618" [label="Lock 0x55b63002b618 (Mutex)" ] "Thread 0x7fac2b0ff700" [label="Thread 0x7fac2b0ff700 (LWP 1123)" ] "Thread 0x7fac46473700" [label="Thread 0x7fac46473700 (LWP 2974)" ] } mongodb-show-locks Mutex at 0x55b6304c0008 held by thread 0x7fac2b0ff700 (LWP 1123) waited on by thread 0x7fac46473700 (LWP 2974) MongoDB Lock at 0x55b630d55c00 (mongo::MODE_X) held by thread id 0x7fac2b0ff700 (LWP 1123) waited on by thread 0x7fac2a8fe700 (LWP 1262) Mutex at 0x55b63002b618 held by thread 0x7fac5179c700 (LWP 32228) waited on by thread 0x7fac2b0ff700 (LWP 1123) Mutex at 0x55b6304c0008 held by thread 0x7fac2b0ff700 (LWP 1123) waited on by thread 0x7fac49f8d700 (LWP 907) MongoDB Lock at 0x55b630d55c00 (mongo::MODE_X) held by thread id 0x7fac2b0ff700 (LWP 1123) waited on by thread 0x7fac48f8b700 (LWP 32342) Mutex at 0x55b6304c0008 held by thread 0x7fac2b0ff700 (LWP 1123) waited on by thread 0x7fac4978c700 (LWP 32341) MongoDB Lock at 0x55b630d55c00 (mongo::MODE_X) held by thread id 0x7fac2b0ff700 (LWP 1123) waited on by thread 0x7fac4b790700 (LWP 32337) Mutex at 0x55b6304c0008 held by thread 0x7fac2b0ff700 (LWP 1123) waited on by thread 0x7fac44782700 (LWP 32323) info threads Id Target Id Frame * 1 Thread 0x7fac54b09c80 (LWP 32216) "mongod" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 2 Thread 0x7fac5179c700 (LWP 32228) "signalP.gThread" 0x00007fac52d4f98d in pthread_join (threadid=140377425176320, thread_return=0x0) at pthread_join.c:90 3 Thread 0x7fac50f9b700 (LWP 32229) "Backgro.kSource" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 4 Thread 0x7fac5079a700 (LWP 32230) "rocksdb:bg0" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 5 Thread 0x7fac4ff99700 (LWP 32231) "rocksdb:bg1" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 6 Thread 0x7fac4f798700 (LWP 32232) "rocksdb:bg2" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 7 Thread 0x7fac4ef97700 (LWP 32233) "rocksdb:bg3" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 8 Thread 0x7fac4e796700 (LWP 32234) "rocksdb:bg4" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 9 Thread 0x7fac4df95700 (LWP 32235) "rocksdb:bg5" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 10 Thread 0x7fac4d794700 (LWP 32236) "rocksdb:bg6" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 11 Thread 0x7fac4cf93700 (LWP 32237) "rocksdb:bg7" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 12 Thread 0x7fac4c792700 (LWP 32238) "rocksdb:bg0" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 13 Thread 0x7fac4bf91700 (LWP 32239) "rocksdb:bg1" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 14 Thread 0x7fac43f81700 (LWP 32322) "RocksCo.nThread" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 15 Thread 0x7fac44782700 (LWP 32323) "RocksJo.Flusher" __lll_lock_wait () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 16 Thread 0x7fac44f83700 (LWP 32335) "DeadlineMonitor" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 17 Thread 0x7fac45784700 (LWP 32336) "Network.cutor-0" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 18 Thread 0x7fac4b790700 (LWP 32337) "ftdc" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 19 Thread 0x7fac4af8f700 (LWP 32338) "monitor.or HMAC" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 20 Thread 0x7fac4a78e700 (LWP 32339) "Network.ation-0" 0x00007fac52a84a13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 21 Thread 0x7fac4978c700 (LWP 32341) "TTLMonitor" __lll_lock_wait () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 22 Thread 0x7fac48f8b700 (LWP 32342) "clientcursormon" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 23 Thread 0x7fac4878a700 (LWP 32343) "Periodi.kRunner" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 24 Thread 0x7fac47f89700 (LWP 32344) "mongod" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 25 Thread 0x7fac46c74700 (LWP 32385) "RocksOp.lThread" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 26 Thread 0x7fac43780700 (LWP 32391) "Network.IO-RS-0" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 27 Thread 0x7fac4277e700 (LWP 32393) "repl wr.orker 0" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 28 Thread 0x7fac41f7d700 (LWP 32394) "repl wr.orker 1" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 29 Thread 0x7fac4177c700 (LWP 32395) "repl wr.orker 2" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 30 Thread 0x7fac40f7b700 (LWP 32396) "repl wr.orker 3" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 31 Thread 0x7fac4077a700 (LWP 32397) "repl wr.orker 4" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 32 Thread 0x7fac3ff79700 (LWP 32398) "repl wr.orker 5" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 33 Thread 0x7fac3f778700 (LWP 32399) "repl wr.orker 6" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 34 Thread 0x7fac3ef77700 (LWP 32400) "repl wr.orker 7" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 35 Thread 0x7fac3e776700 (LWP 32401) "repl wr.orker 8" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 36 Thread 0x7fac3df75700 (LWP 32402) "repl wr.orker 9" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 37 Thread 0x7fac3d774700 (LWP 32403) "repl wr.rker 10" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 38 Thread 0x7fac3cf73700 (LWP 32404) "repl wr.rker 11" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 39 Thread 0x7fac3c772700 (LWP 32405) "repl wr.rker 12" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 40 Thread 0x7fac3bf71700 (LWP 32406) "repl wr.rker 13" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 41 Thread 0x7fac3b770700 (LWP 32407) "repl wr.rker 14" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 42 Thread 0x7fac3af6f700 (LWP 32408) "repl wr.rker 15" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 43 Thread 0x7fac2e756700 (LWP 32448) "Network.ation-0" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 44 Thread 0x7fac49f8d700 (LWP 907) "replexec-8" __lll_lock_wait () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 45 Thread 0x7fac2ff59700 (LWP 1095) "js" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 46 Thread 0x7fac2db51700 (LWP 1096) "js" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 47 Thread 0x7fac2b0ff700 (LWP 1123) "replexec-9" __lll_lock_wait () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 48 Thread 0x7fac2a8fe700 (LWP 1262) "replication-2" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 49 Thread 0x7fac46473700 (LWP 2974) "replexec-10" __lll_lock_wait () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 mongodb-uniqstack Thread 49: "replexec-10" (Thread 0x7fac46473700 (LWP 2974)) Duplicate Thread 44: "replexec-8" (Thread 0x7fac49f8d700 (LWP 907)) #0 __lll_lock_wait () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 #1 0x00007fac52d50dbd in __GI___pthread_mutex_lock (mutex=0x55b6304c0008) at ../nptl/pthread_mutex_lock.c:80 #2 0x000055b62ae8d3d2 in __gthread_mutex_lock (__mutex=0x55b6304c0008) at /usr/include/x86_64-linux-gnu/c++/5/bits/gthr-default.h:748 #3 0x000055b62ae8f470 in std::mutex::lock (this=0x55b6304c0008) at /usr/include/c++/5/mutex:135 #4 0x000055b62ae9192c in std::lock_guard::lock_guard (this=0x7fac46471c90, __m=...) at /usr/include/c++/5/mutex:386 #5 0x000055b62b7a5fd1 in mongo::repl::ReplicationCoordinatorImpl::_doMemberHeartbeat (this=0x55b6304c0000, cbData=..., target=..., targetIndex=2) at src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp:90 #6 0x000055b62b7b4be6 in std::_Mem_fn_base::operator() (this=0x55b63066f900, __object=0x55b6304c0000) at /usr/include/c++/5/functional:600 #7 0x000055b62b7b3d57 in std::_Bind (mongo::repl::ReplicationCoordinatorImpl*, std::_Placeholder<1>, mongo::HostAndPort, int)>::__call(std::tuple&&, std::_Index_tuple<0ul, 1ul, 2ul, 3ul>) (this=0x55b63066f900, __args=) at /usr/include/c++/5/functional:1074 #8 0x000055b62b7b1cea in std::_Bind (mongo::repl::ReplicationCoordinatorImpl*, std::_Placeholder<1>, mongo::HostAndPort, int)>::operator()(mongo::executor::TaskExecutor::CallbackArgs const&) (this=0x55b63066f900) at /usr/include/c++/5/functional:1133 #9 0x000055b62b7afc22 in std::_Function_handler (mongo::repl::ReplicationCoordinatorImpl*, std::_Placeholder<1>, mongo::HostAndPort, int)> >::_M_invoke(std::_Any_data const&, mongo::executor::TaskExecutor::CallbackArgs const&) (__functor=..., __args#0=...) at /usr/include/c++/5/functional:1871 #10 0x000055b62b3c45e1 in std::function::operator()(mongo::executor::TaskExecutor::CallbackArgs const&) const (this=0x7fac46472200, __args#0=...) at /usr/include/c++/5/functional:2267 #11 0x000055b62c66c2d5 in mongo::executor::ThreadPoolTaskExecutor::runCallback (this=0x55b63007ca80, cbStateArg=std::shared_ptr (count 6, weak 0) 0x55b630e66f10) at src/mongo/executor/thread_pool_task_executor.cpp:570 #12 0x000055b62c66bc22 in mongo::executor::ThreadPoolTaskExecutor::::operator()(void) const (__closure=0x55b637959ba0) at src/mongo/executor/thread_pool_task_executor.cpp:546 #13 0x000055b62c66d24f in std::_Function_handler):: >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/5/functional:1871 #14 0x000055b62b13ca24 in std::function::operator()() const (this=0x7fac46472370) at /usr/include/c++/5/functional:2267 #15 0x000055b62b93c97d in mongo::ThreadPool::_doOneTask (this=0x55b6303cb8c0, lk=0x7fac46472470) at src/mongo/util/concurrency/thread_pool.cpp:347 #16 0x000055b62b93c428 in mongo::ThreadPool::_consumeTasks (this=0x55b6303cb8c0) at src/mongo/util/concurrency/thread_pool.cpp:299 #17 0x000055b62b93bdb9 in mongo::ThreadPool::_workerThreadBody (pool=0x55b6303cb8c0, threadName="replexec-10") at src/mongo/util/concurrency/thread_pool.cpp:247 #18 0x000055b62b941a2f in std::_Bind, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)>::__call(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) (this=0x55b6303a8fe8, __args=) at /usr/include/c++/5/functional:1074 #19 0x000055b62b941925 in std::_Bind, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)>::operator()<, void>() (this=0x55b6303a8fe8) at /usr/include/c++/5/functional:1133 #20 0x000055b62b9418ea in std::_Bind_simple, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)> ()>::_M_invoke<>(std::_Index_tuple<>) (this=0x55b6303a8fe8) at /usr/include/c++/5/functional:1531 #21 0x000055b62b941840 in std::_Bind_simple, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)> ()>::operator()() (this=0x55b6303a8fe8) at /usr/include/c++/5/functional:1520 #22 0x000055b62b9417d0 in std::thread::_Impl, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)> ()> >::_M_run() (this=0x55b6303a8fd0) at /usr/include/c++/5/thread:115 #23 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #24 0x00007fac52d4e6ba in start_thread (arg=0x7fac46473700) at pthread_create.c:333 #25 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 48: "replication-2" (Thread 0x7fac2a8fe700 (LWP 1262)) #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x000055b62afa21bd in __gthread_cond_timedwait (__cond=0x55b63b0a49d8, __mutex=0x55b63b0a49b0, __abs_timeout=0x7fac2a8fc810) at /usr/include/x86_64-linux-gnu/c++/5/bits/gthr-default.h:871 #2 0x000055b62afaf4a2 in std::condition_variable::__wait_until_impl > > (this=0x55b63b0a49d8, __lock=..., __atime=...) at /usr/include/c++/5/condition_variable:165 #3 0x000055b62afad4ad in std::condition_variable::wait_until > > (this=0x55b63b0a49d8, __lock=..., __atime=...) at /usr/include/c++/5/condition_variable:105 #4 0x000055b62cc0e4f8 in std::condition_variable::wait_until >, mongo::CondVarLockGrantNotification::wait(mongo::Milliseconds):: >(std::unique_lock &, const std::chrono::time_point > > &, mongo::CondVarLockGrantNotification::) (this=0x55b63b0a49d8, __lock=..., __atime=..., __p=...) at /usr/include/c++/5/condition_variable:128 #5 0x000055b62cc0e49a in std::condition_variable::wait_for, mongo::CondVarLockGrantNotification::wait(mongo::Milliseconds):: >(std::unique_lock &, const std::chrono::duration > &, mongo::CondVarLockGrantNotification::) (this=0x55b63b0a49d8, __lock=..., __rtime=..., __p=...) at /usr/include/c++/5/condit---Type to continue, or q to quit--- ion_variable:144 #6 0x000055b62cc0dcfe in mongo::CondVarLockGrantNotification::wait (this=0x55b63b0a49a8, timeout=...) at src/mongo/db/concurrency/lock_state.cpp:226 #7 0x000055b62cc12fdc in mongo::LockerImpl::lockComplete (this=0x55b63b0a4400, resId=..., mode=mongo::MODE_IX, timeout=..., checkDeadlock=false) at src/mongo/db/concurrency/lock_state.cpp:747 #8 0x000055b62cc1190f in mongo::LockerImpl::lockGlobalComplete (this=0x55b63b0a4400, timeout=...) at src/mongo/db/concurrency/lock_state.cpp:336 #9 0x000055b62cc00f7d in mongo::Lock::GlobalLock::waitForLock (this=0x7fac2a8fcd18, timeoutMs=4294967295) at src/mongo/db/concurrency/d_concurrency.cpp:174 #10 0x000055b62cc00cf7 in mongo::Lock::GlobalLock::GlobalLock (this=0x7fac2a8fcd18, opCtx=0x55b630f36f00, lockMode=mongo::MODE_IX, timeoutMs=4294967295) at src/mongo/db/concurrency/d_concurrency.cpp:141 #11 0x000055b62cc01161 in mongo::Lock::DBLock::DBLock (this=0x7fac2a8fcd00, opCtx=0x55b630f36f00, db="test", mode=mongo::MODE_X) at src/mongo/db/concurrency/d_concurrency.cpp:197 #12 0x000055b62b65913a in mongo::AutoGetDb::AutoGetDb (this=0x7fac2a8fcd00, opCtx=0x55b630f36f00, ns="test", mode=mongo::MODE_X) at src/mongo/db/db_raii.cpp:51 #13 0x000055b62aeca534 in mongo::repl::StorageInterfaceImpl::::operator()(void) const (__closure=0x7fac2a8fce30) at src/mongo/db/repl/storage_interface_impl.cpp:422 #14 0x000055b62aeca7d5 in mongo::writeConflictRetry >(mongo::OperationContext *, mongo::StringData, mongo::StringData, ) (opCtx=0x55b630f36f00, opStr="StorageInterfaceImpl::dropCollection", ns="test.system.drop.1517847163i36t1.upsert_field_gen", f=) at src/mongo/db/concurrency/write_conflict_exception.h:91 #15 0x000055b62aeca930 in mongo::repl::StorageInterfaceImpl::dropCollection (this=0x55b6303c3660, opCtx=0x55b630f36f00, nss=...) at src/mongo/db/repl/storage_interface_impl.cpp:434 #16 0x000055b62b55de1d in mongo::repl::DropPendingCollectionReaper::dropCollectionsOlderThan (this=0x55b6303fde80, opCtx=0x55b630f36f00, opTime=...) at src/mongo/db/repl/drop_pending_collection_reaper.cpp:158 #17 0x000055b62b3c2122 in mongo::repl::ReplicationCoordinatorExternalStateImpl::::operator()(const mongo::executor::TaskExecutor::CallbackArgs &) const (__closure=0x55b630fdede0, args=...) at src/mongo/db/repl/replication_coordinator_external_state_impl.cpp:838 #18 0x000055b62b3c2f36 in std::_Function_handler >::_M_invoke(const std::_Any_data &, const mongo::executor::TaskExecutor::CallbackArgs &) (__functor=..., __args#0=...) at /usr/include/c++/5/functional:1871 #19 0x000055b62b3c45e1 in std::function::operator()(mongo::executor::TaskExecutor::CallbackArgs const&) const (this=0x55b630fde400, __args#0=...) at /usr/include/c++/5/functional:2267 #20 0x000055b62b3bccf9 in mongo::repl::(anonymous namespace)::::operator()(const mongo::executor::TaskExecutor::CallbackArgs &) const (__closure=0x55b630fde400, args=...) at src/mongo/db/repl/replication_coordinator_external_state_impl.cpp:191 #21 0x000055b62b3c2db5 in std::_Function_handler >::_M_invoke(const std::_Any_data &, const mongo::executor::TaskExecutor::CallbackArgs &) (__functor=..., __args#0=...) at /usr/include/c++/5/functional:1871 #22 0x000055b62b3c45e1 in std::function::operator()(mongo::executor::TaskExecutor::CallbackArgs const&) const (this=0x7fac2a8fd200, __args#0=...) at /usr/include/c++/5/functional:2267 #23 0x000055b62c66c2d5 in mongo::executor::ThreadPoolTaskExecutor::runCallback (this=0x55b630ea1340, cbStateArg=std::shared_ptr (count 4, weak 0) 0x55b630dbf0f0) at src/mongo/executor/thread_pool_task_executor.cpp:570 #24 0x000055b62c66bc22 in mongo::executor::ThreadPoolTaskExecutor::::operator()(void) const (__closure=0x55b6379d5520) at src/mongo/executor/thread_pool_task_executor.cpp:546 #25 0x000055b62c66d24f in std::_Function_handler):: >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/5/functional:1871 #26 0x000055b62b13ca24 in std::function::operator()() const (this=0x7fac2a8fd370) at /usr/include/c++/5/functional:2267 #27 0x000055b62b93c97d in mongo::ThreadPool::_doOneTask (this=0x55b6303cca40, lk=0x7fac2a8fd470) at src/mongo/util/concurrency/thread_pool.cpp:347 #28 0x000055b62b93c428 in mongo::ThreadPool::_consumeTasks (this=0x55b6303cca40) at src/mongo/util/concurrency/thread_pool.cpp:299 #29 0x000055b62b93bdb9 in mongo::ThreadPool::_workerThreadBody (pool=0x55b6303cca40, threadName="replication-2") at src/mongo/util/concurrency/thread_pool.cpp:247 #30 0x000055b62b941a2f in std::_Bind, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)>::__call(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) (this=0x55b630f1b268, __args=) at /usr/include/c++/5/functional:1074 #31 0x000055b62b941925 in std::_Bind, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)>::operator()<, void>() (this=0x55b630f1b268) at /usr/include/c++/5/functional:1133 #32 0x000055b62b9418ea in std::_Bind_simple, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)> ()>::_M_invoke<>(std::_Index_tuple<>) (this=0x55b630f1b268) at /usr/include/c++/5/functional:1531 #33 0x000055b62b941840 in std::_Bind_simple, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)> ()>::operator()() (this=0x55b630f1b268) at /usr/include/c++/5/functional:1520 #34 0x000055b62b9417d0 in std::thread::_Impl, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)> ()> >::_M_run() (this=0x55b630f1b250) at /usr/include/c++/5/thread:115 #35 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #36 0x00007fac52d4e6ba in start_thread (arg=0x7fac2a8fe700) at pthread_create.c:333 #37 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 47: "replexec-9" (Thread 0x7fac2b0ff700 (LWP 1123)) #0 __lll_lock_wait () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 #1 0x00007fac52d50dbd in __GI___pthread_mutex_lock (mutex=0x55b63002b618) at ../nptl/pthread_mutex_lock.c:80 #2 0x000055b62ae8d3d2 in __gthread_mutex_lock (__mutex=0x55b63002b618) at /usr/include/x86_64-linux-gnu/c++/5/bits/gthr-default.h:748 #3 0x000055b62ae8f470 in std::mutex::lock (this=0x55b63002b618) at /usr/include/c++/5/mutex:135 #4 0x000055b62ae9192c in std::lock_guard::lock_guard (this=0x7fac2b0fde20, __m=...) at /usr/include/c++/5/mutex:386 ---Type to continue, or q to quit--- #5 0x000055b62b3c1c0d in mongo::repl::ReplicationCoordinatorExternalStateImpl::startProducerIfStopped (this=0x55b63002b600) at src/mongo/db/repl/replication_coordinator_external_state_impl.cpp:781 #6 0x000055b62b78499b in mongo::repl::ReplicationCoordinatorImpl::_updateMemberStateFromTopologyCoordinator_inlock (this=0x55b6304c0000, opCtx=0x55b630655780) at src/mongo/db/repl/replication_coordinator_impl.cpp:2611 #7 0x000055b62b7a8d3f in mongo::repl::ReplicationCoordinatorImpl::_stepDownFinish (this=0x55b6304c0000, cbData=..., finishedEvent=...) at src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp:410 #8 0x000055b62b7b4f6a in std::_Mem_fn_base::operator() (this=0x55b637712260, __object=0x55b6304c0000) at /usr/include/c++/5/functional:600 #9 0x000055b62b7b41b7 in std::_Bind (mongo::repl::ReplicationCoordinatorImpl*, std::_Placeholder<1>, mongo::executor::TaskExecutor::EventHandle)>::__call(std::tuple&&, std::_Index_tuple<0ul, 1ul, 2ul>) (this=0x55b637712260, __args=) at /usr/include/c++/5/functional:1074 #10 0x000055b62b7b2ad2 in std::_Bind (mongo::repl::ReplicationCoordinatorImpl*, std::_Placeholder<1>, mongo::executor::TaskExecutor::EventHandle)>::operator()(mongo::executor::TaskExecutor::CallbackArgs const&) (this=0x55b637712260) at /usr/include/c++/5/functional:1133 #11 0x000055b62b7b07aa in std::_Function_handler (mongo::repl::ReplicationCoordinatorImpl*, std::_Placeholder<1>, mongo::executor::TaskExecutor::EventHandle)> >::_M_invoke(std::_Any_data const&, mongo::executor::TaskExecutor::CallbackArgs const&) (__functor=..., __args#0=...) at /usr/include/c++/5/functional:1871 #12 0x000055b62b3c45e1 in std::function::operator()(mongo::executor::TaskExecutor::CallbackArgs const&) const (this=0x7fac2b0fe200, __args#0=...) at /usr/include/c++/5/functional:2267 #13 0x000055b62c66c2d5 in mongo::executor::ThreadPoolTaskExecutor::runCallback (this=0x55b63007ca80, cbStateArg=std::shared_ptr (count 4, weak 0) 0x55b634768230) at src/mongo/executor/thread_pool_task_executor.cpp:570 #14 0x000055b62c66bc22 in mongo::executor::ThreadPoolTaskExecutor::::operator()(void) const (__closure=0x55b636c61540) at src/mongo/executor/thread_pool_task_executor.cpp:546 #15 0x000055b62c66d24f in std::_Function_handler):: >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/5/functional:1871 #16 0x000055b62b13ca24 in std::function::operator()() const (this=0x7fac2b0fe370) at /usr/include/c++/5/functional:2267 #17 0x000055b62b93c97d in mongo::ThreadPool::_doOneTask (this=0x55b6303cb8c0, lk=0x7fac2b0fe470) at src/mongo/util/concurrency/thread_pool.cpp:347 #18 0x000055b62b93c428 in mongo::ThreadPool::_consumeTasks (this=0x55b6303cb8c0) at src/mongo/util/concurrency/thread_pool.cpp:299 #19 0x000055b62b93bdb9 in mongo::ThreadPool::_workerThreadBody (pool=0x55b6303cb8c0, threadName="replexec-9") at src/mongo/util/concurrency/thread_pool.cpp:247 #20 0x000055b62b941a2f in std::_Bind, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)>::__call(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) (this=0x55b63a53fe08, __args=) at /usr/include/c++/5/functional:1074 #21 0x000055b62b941925 in std::_Bind, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)>::operator()<, void>() (this=0x55b63a53fe08) at /usr/include/c++/5/functional:1133 #22 0x000055b62b9418ea in std::_Bind_simple, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)> ()>::_M_invoke<>(std::_Index_tuple<>) (this=0x55b63a53fe08) at /usr/include/c++/5/functional:1531 #23 0x000055b62b941840 in std::_Bind_simple, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)> ()>::operator()() (this=0x55b63a53fe08) at /usr/include/c++/5/functional:1520 #24 0x000055b62b9417d0 in std::thread::_Impl, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)> ()> >::_M_run() (this=0x55b63a53fdf0) at /usr/include/c++/5/thread:115 #25 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #26 0x00007fac52d4e6ba in start_thread (arg=0x7fac2b0ff700) at pthread_create.c:333 #27 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 46: "js" (Thread 0x7fac2db51700 (LWP 1096)) Duplicate Thread 45: "js" (Thread 0x7fac2ff59700 (LWP 1095)) #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fac5353691c in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x000055b62ba53228 in std::condition_variable::wait >(std::unique_lock &, mongo::mozjs::MozJSProxyScope::) (this=0x55b63112b640, __lock=..., __p=...) at /usr/include/c++/5/condition_variable:98 #3 0x000055b62ba5171b in mongo::mozjs::MozJSProxyScope::implThread (arg=0x55b63112b540) at src/mongo/scripting/mozjs/proxyscope.cpp:357 #4 0x000055b62ba4d646 in nspr::Thread::ThreadRoutine (arg=0x55b63ae847e0) at src/mongo/scripting/mozjs/PosixNSPR.cpp:56 #5 0x000055b62ba4f812 in std::_Bind_simple::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x55b63ae7b4a8) at /usr/include/c++/5/functional:1531 #6 0x000055b62ba4f71c in std::_Bind_simple::operator()() (this=0x55b63ae7b4a8) at /usr/include/c++/5/functional:1520 #7 0x000055b62ba4f6ac in std::thread::_Impl >::_M_run() (this=0x55b63ae7b490) at /usr/include/c++/5/thread:115 #8 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #9 0x00007fac52d4e6ba in start_thread (arg=0x7fac2db51700) at pthread_create.c:333 #10 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 43: "Network.ation-0" (Thread 0x7fac2e756700 (LWP 32448)) #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 ---Type to continue, or q to quit--- #1 0x000055b62c852a36 in asio::detail::posix_event::wait (this=0x55b630ef2370, lock=...) at src/third_party/asio-master/asio/include/asio/detail/posix_event.hpp:105 #2 0x000055b62c851190 in asio::detail::conditionally_enabled_event::wait (this=0x55b630ef2368, lock=...) at src/third_party/asio-master/asio/include/asio/detail/conditionally_enabled_event.hpp:88 #3 0x000055b62c844647 in asio::detail::scheduler::do_run_one (this=0x55b630ef2300, lock=..., this_thread=..., ec=...) at src/third_party/asio-master/asio/include/asio/detail/impl/scheduler.ipp:408 #4 0x000055b62c8437a7 in asio::detail::scheduler::run (this=0x55b630ef2300, ec=...) at src/third_party/asio-master/asio/include/asio/detail/impl/scheduler.ipp:153 #5 0x000055b62c83c5e2 in asio::io_context::run (this=0x55b6305e23b0) at src/third_party/asio-master/asio/include/asio/impl/io_context.ipp:61 #6 0x000055b62c851895 in asio::detail::resolver_service_base::work_io_context_runner::operator() (this=0x55b6305e2d48) at src/third_party/asio-master/asio/include/asio/detail/impl/resolver_service_base.ipp:31 #7 0x000055b62c856ca6 in asio::detail::posix_thread::func::run (this=0x55b6305e2d40) at src/third_party/asio-master/asio/include/asio/detail/posix_thread.hpp:85 #8 0x000055b62c8414e7 in asio::detail::asio_detail_posix_thread_function (arg=0x55b6305e2d40) at src/third_party/asio-master/asio/include/asio/detail/impl/posix_thread.ipp:73 #9 0x00007fac52d4e6ba in start_thread (arg=0x7fac2e756700) at pthread_create.c:333 #10 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 42: "repl wr.rker 15" (Thread 0x7fac3af6f700 (LWP 32408)) Duplicate Thread 41: "repl wr.rker 14" (Thread 0x7fac3b770700 (LWP 32407)) Duplicate Thread 40: "repl wr.rker 13" (Thread 0x7fac3bf71700 (LWP 32406)) Duplicate Thread 39: "repl wr.rker 12" (Thread 0x7fac3c772700 (LWP 32405)) Duplicate Thread 38: "repl wr.rker 11" (Thread 0x7fac3cf73700 (LWP 32404)) Duplicate Thread 37: "repl wr.rker 10" (Thread 0x7fac3d774700 (LWP 32403)) Duplicate Thread 36: "repl wr.orker 9" (Thread 0x7fac3df75700 (LWP 32402)) Duplicate Thread 35: "repl wr.orker 8" (Thread 0x7fac3e776700 (LWP 32401)) Duplicate Thread 34: "repl wr.orker 7" (Thread 0x7fac3ef77700 (LWP 32400)) Duplicate Thread 33: "repl wr.orker 6" (Thread 0x7fac3f778700 (LWP 32399)) Duplicate Thread 32: "repl wr.orker 5" (Thread 0x7fac3ff79700 (LWP 32398)) Duplicate Thread 31: "repl wr.orker 4" (Thread 0x7fac4077a700 (LWP 32397)) Duplicate Thread 30: "repl wr.orker 3" (Thread 0x7fac40f7b700 (LWP 32396)) Duplicate Thread 29: "repl wr.orker 2" (Thread 0x7fac4177c700 (LWP 32395)) Duplicate Thread 28: "repl wr.orker 1" (Thread 0x7fac41f7d700 (LWP 32394)) Duplicate Thread 27: "repl wr.orker 0" (Thread 0x7fac4277e700 (LWP 32393)) #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fac5353691c in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x000055b62b93c3fe in mongo::ThreadPool::_consumeTasks (this=0x55b6303cc880) at src/mongo/util/concurrency/thread_pool.cpp:294 #3 0x000055b62b93bdb9 in mongo::ThreadPool::_workerThreadBody (pool=0x55b6303cc880, threadName="repl writer worker 15") at src/mongo/util/concurrency/thread_pool.cpp:247 #4 0x000055b62b941a2f in std::_Bind, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)>::__call(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) (this=0x55b6303a8088, __args=) at /usr/include/c++/5/functional:1074 #5 0x000055b62b941925 in std::_Bind, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)>::operator()<, void>() (this=0x55b6303a8088) at /usr/include/c++/5/functional:1133 #6 0x000055b62b9418ea in std::_Bind_simple, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)> ()>::_M_invoke<>(std::_Index_tuple<>) (this=0x55b6303a8088) at /usr/include/c++/5/functional:1531 #7 0x000055b62b941840 in std::_Bind_simple, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)> ()>::operator()() (this=0x55b6303a8088) at /usr/include/c++/5/functional:1520 #8 0x000055b62b9417d0 in std::thread::_Impl, std::allocator >))(mongo::ThreadPool*, std::__cxx11::basic_string, std::allocator > const&)> ()> >::_M_run() (this=0x55b6303a8070) at /usr/include/c++/5/thread:115 #9 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #10 0x00007fac52d4e6ba in start_thread (arg=0x7fac3af6f700) at pthread_create.c:333 #11 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 26: "Network.IO-RS-0" (Thread 0x7fac43780700 (LWP 32391)) Duplicate Thread 17: "Network.cutor-0" (Thread 0x7fac45784700 (LWP 32336)) #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x000055b62c852a36 in asio::detail::posix_event::wait (this=0x55b63066f870, lock=...) at src/third_party/asio-master/asio/include/asio/detail/posix_event.hpp:105 #2 0x000055b62c851190 in asio::detail::conditionally_enabled_event::wait (this=0x55b63066f868, lock=...) at src/third_party/asio-master/asio/include/asio/detail/conditionally_enabled_event.hpp:88 #3 0x000055b62c844647 in asio::detail::scheduler::do_run_one (this=0x55b63066f800, lock=..., this_thread=..., ec=...) at src/third_party/asio-master/asio/include/asio/detail/impl/scheduler.ipp:408 #4 0x000055b62c8437a7 in asio::detail::scheduler::run (this=0x55b63066f800, ec=...) at src/third_party/asio-master/asio/include/asio/detail/impl/scheduler.ipp:153 #5 0x000055b62c83c633 in asio::io_context::run (this=0x55b6310d0078, ec=...) at src/third_party/asio-master/asio/include/asio/impl/io_context.ipp:69 #6 0x000055b62c688852 in mongo::executor::NetworkInterfaceASIO::::operator()(void) const (__closure=0x55b630e8b3a8) at src/mongo/executor/network_interface_asio.cpp:160 ---Type to continue, or q to quit--- #7 0x000055b62c68eb2e in std::_Bind_simple()>::_M_invoke<>(std::_Index_tuple<>) (this=0x55b630e8b3a8) at /usr/include/c++/5/functional:1531 #8 0x000055b62c68ea84 in std::_Bind_simple()>::operator()(void) (this=0x55b630e8b3a8) at /usr/include/c++/5/functional:1520 #9 0x000055b62c68ea14 in std::thread::_Impl()> >::_M_run(void) (this=0x55b630e8b390) at /usr/include/c++/5/thread:115 #10 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #11 0x00007fac52d4e6ba in start_thread (arg=0x7fac43780700) at pthread_create.c:333 #12 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 25: "RocksOp.lThread" (Thread 0x7fac46c74700 (LWP 32385)) #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fac5353691c in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x000055b62af64690 in std::condition_variable::wait >(std::unique_lock &, mongo::CappedVisibilityManager::) (this=0x55b630ea0ad8, __lock=..., __p=...) at /usr/include/c++/5/condition_variable:98 #3 0x000055b62af5d462 in mongo::CappedVisibilityManager::oplogJournalThreadLoop (this=0x55b630ea0a80, durabilityManager=0x55b6304f6380) at src/mongo/db/storage/rocks/src/rocks_record_store.cpp:137 #4 0x000055b62af6eef4 in std::_Mem_fn_base::operator()(mongo::CappedVisibilityManager*, mongo::RocksDurabilityManager*&&) const (this=0x55b6306684e8, __object=0x55b630ea0a80) at /usr/include/c++/5/functional:600 #5 0x000055b62af6ee6f in std::_Bind_simple (mongo::CappedVisibilityManager*, mongo::RocksDurabilityManager*)>::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) (this=0x55b6306684d8) at /usr/include/c++/5/functional:1531 #6 0x000055b62af6ed26 in std::_Bind_simple (mongo::CappedVisibilityManager*, mongo::RocksDurabilityManager*)>::operator()() (this=0x55b6306684d8) at /usr/include/c++/5/functional:1520 #7 0x000055b62af6ecb6 in std::thread::_Impl (mongo::CappedVisibilityManager*, mongo::RocksDurabilityManager*)> >::_M_run() (this=0x55b6306684c0) at /usr/include/c++/5/thread:115 #8 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #9 0x00007fac52d4e6ba in start_thread (arg=0x7fac46c74700) at pthread_create.c:333 #10 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 24: "mongod" (Thread 0x7fac47f89700 (LWP 32344)) #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fac5353691c in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x000055b62c5fcce0 in std::condition_variable::wait:: >(std::unique_lock &, mongo::SessionKiller::::) (this=0x55b6305dead0, __lock=..., __p=...) at /usr/include/c++/5/condition_variable:98 #3 0x000055b62c5fbacb in mongo::SessionKiller::SessionKiller(mongo::ServiceContext*, std::function > > (mongo::OperationContext*, mongo::SessionKiller::Matcher const&, std::linear_congruential_engine*)>)::{lambda()#1}::operator()() const () at src/mongo/db/session_killer.cpp:58 #4 0x000055b62c5fde16 in std::_Bind_simple()>::_M_invoke<>(std::_Index_tuple<>) (this=0x55b6304fdaa8) at /usr/include/c++/5/functional:1531 #5 0x000055b62c5fdd6c in std::_Bind_simple()>::operator()(void) (this=0x55b6304fdaa8) at /usr/include/c++/5/functional:1520 #6 0x000055b62c5fdcfc in std::thread::_Impl()> >::_M_run(void) (this=0x55b6304fda90) at /usr/include/c++/5/thread:115 #7 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #8 0x00007fac52d4e6ba in start_thread (arg=0x7fac47f89700) at pthread_create.c:333 #9 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 23: "Periodi.kRunner" (Thread 0x7fac4878a700 (LWP 32343)) #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x000055b62afa21bd in __gthread_cond_timedwait (__cond=0x55b630089800, __mutex=0x55b6300897d8, __abs_timeout=0x7fac48789610) at /usr/include/x86_64-linux-gnu/c++/5/bits/gthr-default.h:871 #2 0x000055b62afaf4a2 in std::condition_variable::__wait_until_impl > > (this=0x55b630089800, __lock=..., __atime=...) at /usr/include/c++/5/condition_variable:165 #3 0x000055b62afad4ad in std::condition_variable::wait_until > > (this=0x55b630089800, __lock=..., __atime=...) at /usr/include/c++/5/condition_variable:105 #4 0x000055b62b3d494e in std::condition_variable::wait_for > (this=0x55b630089800, __lock=..., __rtime=...) at /usr/include/c++/5/condition_variable:137 #5 0x000055b62cc7bfe9 in mongo::(anonymous namespace)::PeriodicTaskRunner::run (this=0x55b6300897c0) at src/mongo/util/background.cpp:313 #6 0x000055b62cc7b149 in mongo::BackgroundJob::jobBody (this=0x55b6300897c0) at src/mongo/util/background.cpp:150 #7 0x000055b62cc7e9d1 in std::_Mem_fn_base::operator()<, void>(mongo::BackgroundJob*) const (this=0x55b6304fde28, __object=0x55b6300897c0) at /usr/include/c++/5/functional:600 #8 0x000055b62cc7e925 in std::_Bind (mongo::BackgroundJob*)>::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x55b6304fde28, __args=) at /usr/include/c++/5/functional:1074 #9 0x000055b62cc7e87b in std::_Bind (mongo::BackgroundJob*)>::operator()<, void>() (this=0x55b6304fde28) at /usr/include/c++/5/functional:1133 #10 0x000055b62cc7e840 in std::_Bind_simple (mongo::BackgroundJob*)> ()>::_M_invoke<>(std::_Index_tuple<>) (this=0x55b6304fde28) at /usr/include/c++/5/functional:1531 ---Type to continue, or q to quit--- #11 0x000055b62cc7e796 in std::_Bind_simple (mongo::BackgroundJob*)> ()>::operator()() (this=0x55b6304fde28) at /usr/include/c++/5/functional:1520 #12 0x000055b62cc7e726 in std::thread::_Impl (mongo::BackgroundJob*)> ()> >::_M_run() (this=0x55b6304fde10) at /usr/include/c++/5/thread:115 #13 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #14 0x00007fac52d4e6ba in start_thread (arg=0x7fac4878a700) at pthread_create.c:333 #15 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 22: "clientcursormon" (Thread 0x7fac48f8b700 (LWP 32342)) #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x000055b62afa21bd in __gthread_cond_timedwait (__cond=0x55b630e921d8, __mutex=0x55b630e921b0, __abs_timeout=0x7fac48f89fd0) at /usr/include/x86_64-linux-gnu/c++/5/bits/gthr-default.h:871 #2 0x000055b62afaf4a2 in std::condition_variable::__wait_until_impl > > (this=0x55b630e921d8, __lock=..., __atime=...) at /usr/include/c++/5/condition_variable:165 #3 0x000055b62afad4ad in std::condition_variable::wait_until > > (this=0x55b630e921d8, __lock=..., __atime=...) at /usr/include/c++/5/condition_variable:105 #4 0x000055b62cc0e4f8 in std::condition_variable::wait_until >, mongo::CondVarLockGrantNotification::wait(mongo::Milliseconds):: >(std::unique_lock &, const std::chrono::time_point > > &, mongo::CondVarLockGrantNotification::) (this=0x55b630e921d8, __lock=..., __atime=..., __p=...) at /usr/include/c++/5/condition_variable:128 #5 0x000055b62cc0e49a in std::condition_variable::wait_for, mongo::CondVarLockGrantNotification::wait(mongo::Milliseconds):: >(std::unique_lock &, const std::chrono::duration > &, mongo::CondVarLockGrantNotification::) (this=0x55b630e921d8, __lock=..., __rtime=..., __p=...) at /usr/include/c++/5/condition_variable:144 #6 0x000055b62cc0dcfe in mongo::CondVarLockGrantNotification::wait (this=0x55b630e921a8, timeout=...) at src/mongo/db/concurrency/lock_state.cpp:226 #7 0x000055b62cc12fdc in mongo::LockerImpl::lockComplete (this=0x55b630e91c00, resId=..., mode=mongo::MODE_IS, timeout=..., checkDeadlock=false) at src/mongo/db/concurrency/lock_state.cpp:747 #8 0x000055b62cc1190f in mongo::LockerImpl::lockGlobalComplete (this=0x55b630e91c00, timeout=...) at src/mongo/db/concurrency/lock_state.cpp:336 #9 0x000055b62cc00f7d in mongo::Lock::GlobalLock::waitForLock (this=0x7fac48f8a468, timeoutMs=4294967295) at src/mongo/db/concurrency/d_concurrency.cpp:174 #10 0x000055b62cc00cf7 in mongo::Lock::GlobalLock::GlobalLock (this=0x7fac48f8a468, opCtx=0x55b630f36640, lockMode=mongo::MODE_IS, timeoutMs=4294967295) at src/mongo/db/concurrency/d_concurrency.cpp:141 #11 0x000055b62cc01161 in mongo::Lock::DBLock::DBLock (this=0x7fac48f8a450, opCtx=0x55b630f36640, db="local", mode=mongo::MODE_IS) at src/mongo/db/concurrency/d_concurrency.cpp:197 #12 0x000055b62b65a6a6 in mongo::AutoGetCollectionForReadCommand::AutoGetCollectionForReadCommand (this=0x7fac48f8a5f0, opCtx=0x55b630f36640, nss=..., viewMode=mongo::AutoGetCollection::ViewMode::kViewsPermitted) at src/mongo/db/db_raii.cpp:247 #13 0x000055b62b65a745 in mongo::AutoGetCollectionOrViewForReadCommand::AutoGetCollectionOrViewForReadCommand (this=0x7fac48f8a5f0, opCtx=0x55b630f36640, nss=...) at src/mongo/db/db_raii.cpp:254 #14 0x000055b62b566896 in mongo::(anonymous namespace)::GlobalCursorIdCache::timeoutCursors (this=0x55b6303fdd00, opCtx=0x55b630f36640, now=...) at src/mongo/db/cursor_manager.cpp:254 #15 0x000055b62b566da2 in mongo::CursorManager::timeoutCursorsGlobal (opCtx=0x55b630f36640, now=...) at src/mongo/db/cursor_manager.cpp:333 #16 0x000055b62b5650eb in mongo::(anonymous namespace)::ClientCursorMonitor::run (this=0x55b62e693bd0 ) at src/mongo/db/clientcursor.cpp:284 #17 0x000055b62cc7b149 in mongo::BackgroundJob::jobBody (this=0x55b62e693bd0 ) at src/mongo/util/background.cpp:150 #18 0x000055b62cc7e9d1 in std::_Mem_fn_base::operator()<, void>(mongo::BackgroundJob*) const (this=0x55b6304fe0e8, __object=0x55b62e693bd0 ) at /usr/include/c++/5/functional:600 #19 0x000055b62cc7e925 in std::_Bind (mongo::BackgroundJob*)>::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x55b6304fe0e8, __args=) at /usr/include/c++/5/functional:1074 #20 0x000055b62cc7e87b in std::_Bind (mongo::BackgroundJob*)>::operator()<, void>() (this=0x55b6304fe0e8) at /usr/include/c++/5/functional:1133 #21 0x000055b62cc7e840 in std::_Bind_simple (mongo::BackgroundJob*)> ()>::_M_invoke<>(std::_Index_tuple<>) (this=0x55b6304fe0e8) at /usr/include/c++/5/functional:1531 #22 0x000055b62cc7e796 in std::_Bind_simple (mongo::BackgroundJob*)> ()>::operator()() (this=0x55b6304fe0e8) at /usr/include/c++/5/functional:1520 #23 0x000055b62cc7e726 in std::thread::_Impl (mongo::BackgroundJob*)> ()> >::_M_run() (this=0x55b6304fe0d0) at /usr/include/c++/5/thread:115 #24 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #25 0x00007fac52d4e6ba in start_thread (arg=0x7fac48f8b700) at pthread_create.c:333 #26 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 21: "TTLMonitor" (Thread 0x7fac4978c700 (LWP 32341)) #0 __lll_lock_wait () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 #1 0x00007fac52d50dbd in __GI___pthread_mutex_lock (mutex=0x55b6304c0008) at ../nptl/pthread_mutex_lock.c:80 #2 0x000055b62ae8d3d2 in __gthread_mutex_lock (__mutex=0x55b6304c0008) at /usr/include/x86_64-linux-gnu/c++/5/bits/gthr-default.h:748 #3 0x000055b62ae8f470 in std::mutex::lock (this=0x55b6304c0008) at /usr/include/c++/5/mutex:135 #4 0x000055b62ae9192c in std::lock_guard::lock_guard (this=0x7fac4978b3c0, __m=...) at /usr/include/c++/5/mutex:386 #5 0x000055b62b77907b in mongo::repl::ReplicationCoordinatorImpl::getMemberState (this=0x55b6304c0000) at src/mongo/db/repl/replication_coordinator_impl.cpp:838 #6 0x000055b62b1d7e54 in mongo::TTLMonitor::doTTLPass (this=0x55b6303c1580) at src/mongo/db/ttl.cpp:126 #7 0x000055b62b1d7c0e in mongo::TTLMonitor::run (this=0x55b6303c1580) at src/mongo/db/ttl.cpp:111 #8 0x000055b62cc7b149 in mongo::BackgroundJob::jobBody (this=0x55b6303c1580) at src/mongo/util/background.cpp:150 ---Type to continue, or q to quit--- #9 0x000055b62cc7e9d1 in std::_Mem_fn_base::operator()<, void>(mongo::BackgroundJob*) const (this=0x55b6304fdca8, __object=0x55b6303c1580) at /usr/include/c++/5/functional:600 #10 0x000055b62cc7e925 in std::_Bind (mongo::BackgroundJob*)>::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x55b6304fdca8, __args=) at /usr/include/c++/5/functional:1074 #11 0x000055b62cc7e87b in std::_Bind (mongo::BackgroundJob*)>::operator()<, void>() (this=0x55b6304fdca8) at /usr/include/c++/5/functional:1133 #12 0x000055b62cc7e840 in std::_Bind_simple (mongo::BackgroundJob*)> ()>::_M_invoke<>(std::_Index_tuple<>) (this=0x55b6304fdca8) at /usr/include/c++/5/functional:1531 #13 0x000055b62cc7e796 in std::_Bind_simple (mongo::BackgroundJob*)> ()>::operator()() (this=0x55b6304fdca8) at /usr/include/c++/5/functional:1520 #14 0x000055b62cc7e726 in std::thread::_Impl (mongo::BackgroundJob*)> ()> >::_M_run() (this=0x55b6304fdc90) at /usr/include/c++/5/thread:115 #15 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #16 0x00007fac52d4e6ba in start_thread (arg=0x7fac4978c700) at pthread_create.c:333 #17 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 20: "Network.ation-0" (Thread 0x7fac4a78e700 (LWP 32339)) #0 0x00007fac52a84a13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 #1 0x000055b62c84027a in asio::detail::epoll_reactor::run (this=0x55b630ea1ea0, usec=-1, ops=...) at src/third_party/asio-master/asio/include/asio/detail/impl/epoll_reactor.ipp:452 #2 0x000055b62c844582 in asio::detail::scheduler::do_run_one (this=0x55b6303aa600, lock=..., this_thread=..., ec=...) at src/third_party/asio-master/asio/include/asio/detail/impl/scheduler.ipp:384 #3 0x000055b62c8437a7 in asio::detail::scheduler::run (this=0x55b6303aa600, ec=...) at src/third_party/asio-master/asio/include/asio/detail/impl/scheduler.ipp:153 #4 0x000055b62c83c633 in asio::io_context::run (this=0x55b6304be078, ec=...) at src/third_party/asio-master/asio/include/asio/impl/io_context.ipp:69 #5 0x000055b62c688852 in mongo::executor::NetworkInterfaceASIO::::operator()(void) const (__closure=0x55b6304f5c68) at src/mongo/executor/network_interface_asio.cpp:160 #6 0x000055b62c68eb2e in std::_Bind_simple()>::_M_invoke<>(std::_Index_tuple<>) (this=0x55b6304f5c68) at /usr/include/c++/5/functional:1531 #7 0x000055b62c68ea84 in std::_Bind_simple()>::operator()(void) (this=0x55b6304f5c68) at /usr/include/c++/5/functional:1520 #8 0x000055b62c68ea14 in std::thread::_Impl()> >::_M_run(void) (this=0x55b6304f5c50) at /usr/include/c++/5/thread:115 #9 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #10 0x00007fac52d4e6ba in start_thread (arg=0x7fac4a78e700) at pthread_create.c:333 #11 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 19: "monitor.or HMAC" (Thread 0x7fac4af8f700 (LWP 32338)) #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x000055b62afa21bd in __gthread_cond_timedwait (__cond=0x55b6305e3708, __mutex=0x55b6305e36d0, __abs_timeout=0x7fac4af8e4d0) at /usr/include/x86_64-linux-gnu/c++/5/bits/gthr-default.h:871 #2 0x000055b62afaf4a2 in std::condition_variable::__wait_until_impl > > (this=0x55b6305e3708, __lock=..., __atime=...) at /usr/include/c++/5/condition_variable:165 #3 0x000055b62afad4ad in std::condition_variable::wait_until > > (this=0x55b6305e3708, __lock=..., __atime=...) at /usr/include/c++/5/condition_variable:105 #4 0x000055b62cc3516a in mongo::ClockSource::waitForConditionUntil (this=0x55b630024d40, cv=..., m=..., deadline=...) at src/mongo/util/clock_source.cpp:39 #5 0x000055b62cc2790e in mongo::OperationContext::::operator()(void) const (__closure=0x7fac4af8e6b0) at src/mongo/db/operation_context.cpp:325 #6 0x000055b62cc27c59 in mongo::OperationContext::waitForConditionOrInterruptNoAssertUntil (this=0x55b630656540, cv=..., m=..., deadline=...) at src/mongo/db/operation_context.cpp:326 #7 0x000055b62c8612b1 in mongo::KeysCollectionManagerSharding::PeriodicRunner::_doPeriodicRefresh (this=0x55b6305e36d0, service=0x55b63002b480, threadName="monitoring keys for HMAC", refreshInterval=...) at src/mongo/db/keys_collection_manager_sharding.cpp:294 #8 0x000055b62c865d36 in std::_Mem_fn_base, std::allocator >, mongo::Duration >), true>::operator(), std::allocator >&, mongo::Duration >&, void> (this=0x55b63064f928, __object=0x55b6305e36d0) at /usr/include/c++/5/functional:600 #9 0x000055b62c865b7d in std::_Bind, std::allocator >, mongo::Duration >)> (mongo::KeysCollectionManagerSharding::PeriodicRunner*, mongo::ServiceContext*, std::__cxx11::basic_string, std::allocator >, mongo::Duration >)>::__call(std::tuple<>&&, std::_Index_tuple<0ul, 1ul, 2ul, 3ul>) (this=0x55b63064f928, __args=) at /usr/include/c++/5/functional:1074 #10 0x000055b62c86593b in std::_Bind, std::allocator >, mongo::Duration >)> (mongo::KeysCollectionManagerSharding::PeriodicRunner*, mongo::ServiceContext*, std::__cxx11::basic_string, std::allocator >, mongo::Duration >)>::operator()<, void>() (this=0x55b63064f928) at /usr/include/c++/5/functional:1133 #11 0x000055b62c865896 in std::_Bind_simple, std::allocator >, mongo::Duration >)> (mongo::KeysCollectionManagerSharding::PeriodicRunner*, mongo::ServiceContext*, std::__cxx11::basic_string, std::allocator >, mongo::Duration >)> ()>::_M_invoke<>(std::_Index_tuple<>) (this=0x55b63064f928) at /usr/include/c++/5/functional:1531 #12 0x000055b62c86572a in std::_Bind_simple, std::allocator >, mongo::Duration >)> (mongo::KeysCollectionManagerSharding::PeriodicRunner*, mongo::ServiceContext*, std::__cxx11::basic_string, std::allocator >, mongo::Duration >)> ()>::operator()() (this=0x55b63064f928) at /usr/include/c++/5/functional:1520 #13 0x000055b62c865624 in std::thread::_Impl, std::allocator >, mongo::Duration >)> (mongo::KeysCollectionManagerSharding::PeriodicRunner*, mongo::ServiceContext*, std::__cxx11::basic_string to continue, or q to quit--- , std::char_traits, std::allocator >, mongo::Duration >)> ()> >::_M_run() (this=0x55b63064f910) at /usr/include/c++/5/thread:115 #14 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #15 0x00007fac52d4e6ba in start_thread (arg=0x7fac4af8f700) at pthread_create.c:333 #16 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 18: "ftdc" (Thread 0x7fac4b790700 (LWP 32337)) #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x000055b62afa21bd in __gthread_cond_timedwait (__cond=0x55b630e939d8, __mutex=0x55b630e939b0, __abs_timeout=0x7fac4b78e2e0) at /usr/include/x86_64-linux-gnu/c++/5/bits/gthr-default.h:871 #2 0x000055b62afaf4a2 in std::condition_variable::__wait_until_impl > > (this=0x55b630e939d8, __lock=..., __atime=...) at /usr/include/c++/5/condition_variable:165 #3 0x000055b62afad4ad in std::condition_variable::wait_until > > (this=0x55b630e939d8, __lock=..., __atime=...) at /usr/include/c++/5/condition_variable:105 #4 0x000055b62cc0e4f8 in std::condition_variable::wait_until >, mongo::CondVarLockGrantNotification::wait(mongo::Milliseconds):: >(std::unique_lock &, const std::chrono::time_point > > &, mongo::CondVarLockGrantNotification::) (this=0x55b630e939d8, __lock=..., __atime=..., __p=...) at /usr/include/c++/5/condition_variable:128 #5 0x000055b62cc0e49a in std::condition_variable::wait_for, mongo::CondVarLockGrantNotification::wait(mongo::Milliseconds):: >(std::unique_lock &, const std::chrono::duration > &, mongo::CondVarLockGrantNotification::) (this=0x55b630e939d8, __lock=..., __rtime=..., __p=...) at /usr/include/c++/5/condition_variable:144 #6 0x000055b62cc0dcfe in mongo::CondVarLockGrantNotification::wait (this=0x55b630e939a8, timeout=...) at src/mongo/db/concurrency/lock_state.cpp:226 #7 0x000055b62cc12fdc in mongo::LockerImpl::lockComplete (this=0x55b630e93400, resId=..., mode=mongo::MODE_IS, timeout=..., checkDeadlock=false) at src/mongo/db/concurrency/lock_state.cpp:747 #8 0x000055b62cc1190f in mongo::LockerImpl::lockGlobalComplete (this=0x55b630e93400, timeout=...) at src/mongo/db/concurrency/lock_state.cpp:336 #9 0x000055b62cc00f7d in mongo::Lock::GlobalLock::waitForLock (this=0x7fac4b78e8b0, timeoutMs=4294967295) at src/mongo/db/concurrency/d_concurrency.cpp:174 #10 0x000055b62cc00cf7 in mongo::Lock::GlobalLock::GlobalLock (this=0x7fac4b78e8b0, opCtx=0x55b630e9b400, lockMode=mongo::MODE_IS, timeoutMs=4294967295) at src/mongo/db/concurrency/d_concurrency.cpp:141 #11 0x000055b62af066ed in mongo::RocksServerStatusSection::generateSection (this=0x55b630392a10, opCtx=0x55b630e9b400, configElement=...) at src/mongo/db/storage/rocks/src/rocks_server_status.cpp:113 #12 0x000055b62ae90d62 in mongo::ServerStatusSection::appendSection (this=0x55b630392a10, opCtx=0x55b630e9b400, configElement=..., result=0x7fac4b78f490) at src/mongo/db/commands/server_status.h:101 #13 0x000055b62c60ad2a in mongo::CmdServerStatus::run (this=0x55b62e6a9620 , opCtx=0x55b630e9b400, dbname="", cmdObj=owned BSONObj 55 bytes @ 0x55b6306699a8, result=...) at src/mongo/db/commands/server_status.cpp:139 #14 0x000055b62c79b9eb in mongo::BasicCommand::enhancedRun (this=0x55b62e6a9620 , opCtx=0x55b630e9b400, request=..., result=...) at src/mongo/db/commands.cpp:416 #15 0x000055b62c79b475 in mongo::Command::publicRun (this=0x55b62e6a9620 , opCtx=0x55b630e9b400, request=..., result=...) at src/mongo/db/commands.cpp:354 #16 0x000055b62c79a209 in mongo::Command::runCommandDirectly (opCtx=0x55b630e9b400, request=...) at src/mongo/db/commands.cpp:218 #17 0x000055b62b3264cb in mongo::FTDCSimpleInternalCommandCollector::collect (this=0x55b630669900, opCtx=0x55b630e9b400, builder=...) at src/mongo/db/ftdc/ftdc_server.cpp:256 #18 0x000055b62b332f0e in mongo::FTDCCollectorCollection::collect (this=0x55b6306555d0, client=0x55b63066c680) at src/mongo/db/ftdc/collector.cpp:86 #19 0x000055b62b338b20 in mongo::FTDCController::doLoop (this=0x55b630655500) at src/mongo/db/ftdc/controller.cpp:244 #20 0x000055b62b33ad13 in std::_Mem_fn_base::operator()<, void>(mongo::FTDCController*) const (this=0x55b6304f59a8, __object=0x55b630655500) at /usr/include/c++/5/functional:600 #21 0x000055b62b33ac75 in std::_Bind (mongo::FTDCController*)>::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x55b6304f59a8, __args=) at /usr/include/c++/5/functional:1074 #22 0x000055b62b33abcb in std::_Bind (mongo::FTDCController*)>::operator()<, void>() (this=0x55b6304f59a8) at /usr/include/c++/5/functional:1133 #23 0x000055b62b33ab90 in std::_Bind_simple (mongo::FTDCController*)> ()>::_M_invoke<>(std::_Index_tuple<>) (this=0x55b6304f59a8) at /usr/include/c++/5/functional:1531 #24 0x000055b62b33aae6 in std::_Bind_simple (mongo::FTDCController*)> ()>::operator()() (this=0x55b6304f59a8) at /usr/include/c++/5/functional:1520 #25 0x000055b62b33aa76 in std::thread::_Impl (mongo::FTDCController*)> ()> >::_M_run() (this=0x55b6304f5990) at /usr/include/c++/5/thread:115 #26 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #27 0x00007fac52d4e6ba in start_thread (arg=0x7fac4b790700) at pthread_create.c:333 #28 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 16: "DeadlineMonitor" (Thread 0x7fac44f83700 (LWP 32335)) #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x000055b62afa21bd in __gthread_cond_timedwait (__cond=0x55b630655990, __mutex=0x55b630655968, __abs_timeout=0x7fac44f827a0) at /usr/include/x86_64-linux-gnu/c++/5/bits/gthr-default.h:871 #2 0x000055b62afaf4a2 in std::condition_variable::__wait_until_impl > > (this=0x55b630655990, __lock=..., __atime=...) at /usr/include/c++/5/condition_variable:165 #3 0x000055b62afad4ad in std::condition_variable::wait_until > > (this=0x55b630655990, __lock=..., __atime=...) at /usr/include/c++/5/condition_variable:105 #4 0x000055b62b3d494e in std::condition_variable::wait_for > (this=0x55b630655990, __lock=..., __rtime=...) at /usr/include/c++/5/condition_variable:137 #5 0x000055b62ba00668 in mongo::DeadlineMonitor::deadlineMonitorThread (this=0x55b630655930) at src/mongo/scripting/deadline_monitor.h:158 ---Type to continue, or q to quit--- #6 0x000055b62ba040ad in std::_Mem_fn_base::*)(), true>::operator()<, void>(mongo::DeadlineMonitor*) const (this=0x55b6304f6070, __object=0x55b630655930) at /usr/include/c++/5/functional:600 #7 0x000055b62ba04041 in std::_Bind_simple::*)()> (mongo::DeadlineMonitor*)>::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x55b6304f6068) at /usr/include/c++/5/functional:1531 #8 0x000055b62ba03f48 in std::_Bind_simple::*)()> (mongo::DeadlineMonitor*)>::operator()() (this=0x55b6304f6068) at /usr/include/c++/5/functional:1520 #9 0x000055b62ba03ed8 in std::thread::_Impl::*)()> (mongo::DeadlineMonitor*)> >::_M_run() (this=0x55b6304f6050) at /usr/include/c++/5/thread:115 #10 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #11 0x00007fac52d4e6ba in start_thread (arg=0x7fac44f83700) at pthread_create.c:333 #12 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 15: "RocksJo.Flusher" (Thread 0x7fac44782700 (LWP 32323)) #0 __lll_lock_wait () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 #1 0x00007fac52d50dbd in __GI___pthread_mutex_lock (mutex=0x55b6304c0008) at ../nptl/pthread_mutex_lock.c:80 #2 0x000055b62ae8d3d2 in __gthread_mutex_lock (__mutex=0x55b6304c0008) at /usr/include/x86_64-linux-gnu/c++/5/bits/gthr-default.h:748 #3 0x000055b62ae8f470 in std::mutex::lock (this=0x55b6304c0008) at /usr/include/c++/5/mutex:135 #4 0x000055b62af40bc1 in std::unique_lock::lock (this=0x7fac44781540) at /usr/include/c++/5/mutex:485 #5 0x000055b62af400e6 in std::unique_lock::unique_lock (this=0x7fac44781540, __m=...) at /usr/include/c++/5/mutex:415 #6 0x000055b62b77a50f in mongo::repl::ReplicationCoordinatorImpl::setMyLastDurableOpTimeForward (this=0x55b6304c0000, opTime=...) at src/mongo/db/repl/replication_coordinator_impl.cpp:1063 #7 0x000055b62b3c2888 in mongo::repl::ReplicationCoordinatorExternalStateImpl::onDurable (this=0x55b63002b600, token=...) at src/mongo/db/repl/replication_coordinator_external_state_impl.cpp:908 #8 0x000055b62af8311b in mongo::RocksDurabilityManager::waitUntilDurable (this=0x55b6304f6380, forceFlush=false) at src/mongo/db/storage/rocks/src/rocks_durability_manager.cpp:53 #9 0x000055b62af50a94 in mongo::RocksEngine::RocksJournalFlusher::run (this=0x55b6303c0ce0) at src/mongo/db/storage/rocks/src/rocks_engine.cpp:94 #10 0x000055b62cc7b149 in mongo::BackgroundJob::jobBody (this=0x55b6303c0ce0) at src/mongo/util/background.cpp:150 #11 0x000055b62cc7e9d1 in std::_Mem_fn_base::operator()<, void>(mongo::BackgroundJob*) const (this=0x55b6304f62e8, __object=0x55b6303c0ce0) at /usr/include/c++/5/functional:600 #12 0x000055b62cc7e925 in std::_Bind (mongo::BackgroundJob*)>::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x55b6304f62e8, __args=) at /usr/include/c++/5/functional:1074 #13 0x000055b62cc7e87b in std::_Bind (mongo::BackgroundJob*)>::operator()<, void>() (this=0x55b6304f62e8) at /usr/include/c++/5/functional:1133 #14 0x000055b62cc7e840 in std::_Bind_simple (mongo::BackgroundJob*)> ()>::_M_invoke<>(std::_Index_tuple<>) (this=0x55b6304f62e8) at /usr/include/c++/5/functional:1531 #15 0x000055b62cc7e796 in std::_Bind_simple (mongo::BackgroundJob*)> ()>::operator()() (this=0x55b6304f62e8) at /usr/include/c++/5/functional:1520 #16 0x000055b62cc7e726 in std::thread::_Impl (mongo::BackgroundJob*)> ()> >::_M_run() (this=0x55b6304f62d0) at /usr/include/c++/5/thread:115 #17 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #18 0x00007fac52d4e6ba in start_thread (arg=0x7fac44782700) at pthread_create.c:333 #19 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 14: "RocksCo.nThread" (Thread 0x7fac43f81700 (LWP 32322)) #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fac5353691c in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x000055b62af3b5fc in mongo::CompactionBackgroundJob::run (this=0x55b6304073c0) at src/mongo/db/storage/rocks/src/rocks_compaction_scheduler.cpp:223 #3 0x000055b62cc7b149 in mongo::BackgroundJob::jobBody (this=0x55b6304073c0) at src/mongo/util/background.cpp:150 #4 0x000055b62cc7e9d1 in std::_Mem_fn_base::operator()<, void>(mongo::BackgroundJob*) const (this=0x55b6304f64e8, __object=0x55b6304073c0) at /usr/include/c++/5/functional:600 #5 0x000055b62cc7e925 in std::_Bind (mongo::BackgroundJob*)>::__call(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x55b6304f64e8, __args=) at /usr/include/c++/5/functional:1074 #6 0x000055b62cc7e87b in std::_Bind (mongo::BackgroundJob*)>::operator()<, void>() (this=0x55b6304f64e8) at /usr/include/c++/5/functional:1133 #7 0x000055b62cc7e840 in std::_Bind_simple (mongo::BackgroundJob*)> ()>::_M_invoke<>(std::_Index_tuple<>) (this=0x55b6304f64e8) at /usr/include/c++/5/functional:1531 #8 0x000055b62cc7e796 in std::_Bind_simple (mongo::BackgroundJob*)> ()>::operator()() (this=0x55b6304f64e8) at /usr/include/c++/5/functional:1520 #9 0x000055b62cc7e726 in std::thread::_Impl (mongo::BackgroundJob*)> ()> >::_M_run() (this=0x55b6304f64d0) at /usr/include/c++/5/thread:115 #10 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #11 0x00007fac52d4e6ba in start_thread (arg=0x7fac43f81700) at pthread_create.c:333 #12 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 ---Type to continue, or q to quit--- Thread 13: "rocksdb:bg1" (Thread 0x7fac4bf91700 (LWP 32239)) Duplicate Thread 12: "rocksdb:bg0" (Thread 0x7fac4c792700 (LWP 32238)) Duplicate Thread 11: "rocksdb:bg7" (Thread 0x7fac4cf93700 (LWP 32237)) Duplicate Thread 10: "rocksdb:bg6" (Thread 0x7fac4d794700 (LWP 32236)) Duplicate Thread 9: "rocksdb:bg5" (Thread 0x7fac4df95700 (LWP 32235)) Duplicate Thread 8: "rocksdb:bg4" (Thread 0x7fac4e796700 (LWP 32234)) Duplicate Thread 7: "rocksdb:bg3" (Thread 0x7fac4ef97700 (LWP 32233)) Duplicate Thread 6: "rocksdb:bg2" (Thread 0x7fac4f798700 (LWP 32232)) Duplicate Thread 5: "rocksdb:bg1" (Thread 0x7fac4ff99700 (LWP 32231)) Duplicate Thread 4: "rocksdb:bg0" (Thread 0x7fac5079a700 (LWP 32230)) #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fac5353691c in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x000055b62d0737de in rocksdb::ThreadPoolImpl::Impl::BGThread (this=this@entry=0x55b6305bb340, thread_id=thread_id@entry=1) at util/threadpool_imp.cc:180 #3 0x000055b62d073b6d in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper (arg=0x55b6305e17f0) at util/threadpool_imp.cc:261 #4 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #5 0x00007fac52d4e6ba in start_thread (arg=0x7fac4bf91700) at pthread_create.c:333 #6 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 3: "Backgro.kSource" (Thread 0x7fac50f9b700 (LWP 32229)) #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fac5353691c in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x000055b62cc33980 in std::condition_variable::wait:: >(std::unique_lock &, mongo::BackgroundThreadClockSource::::) (this=0x55b6305ea638, __lock=..., __p=...) at /usr/include/c++/5/condition_variable:98 #3 0x000055b62cc335a2 in mongo::BackgroundThreadClockSource::::operator()(void) const (__closure=0x55b6303929d8) at src/mongo/util/background_thread_clock_source.cpp:120 #4 0x000055b62cc34af8 in std::_Bind_simple()>::_M_invoke<>(std::_Index_tuple<>) (this=0x55b6303929d8) at /usr/include/c++/5/functional:1531 #5 0x000055b62cc34a4e in std::_Bind_simple()>::operator()(void) (this=0x55b6303929d8) at /usr/include/c++/5/functional:1520 #6 0x000055b62cc349de in std::thread::_Impl()> >::_M_run(void) (this=0x55b6303929c0) at /usr/include/c++/5/thread:115 #7 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #8 0x00007fac52d4e6ba in start_thread (arg=0x7fac50f9b700) at pthread_create.c:333 #9 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 2: "signalP.gThread" (Thread 0x7fac5179c700 (LWP 32228)) #0 0x00007fac52d4f98d in pthread_join (threadid=140377425176320, thread_return=0x0) at pthread_join.c:90 #1 0x00007fac5353bb97 in std::thread::join() () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x000055b62b93b55d in mongo::ThreadPool::_join_inlock (this=0x55b6303cca40, lk=0x7fac5179af90) at src/mongo/util/concurrency/thread_pool.cpp:170 #3 0x000055b62b93b1c7 in mongo::ThreadPool::join (this=0x55b6303cca40) at src/mongo/util/concurrency/thread_pool.cpp:134 #4 0x000055b62c668d2b in mongo::executor::ThreadPoolTaskExecutor::_join (this=0x55b630ea1340, lk=...) at src/mongo/executor/thread_pool_task_executor.cpp:198 #5 0x000055b62c668b88 in mongo::executor::ThreadPoolTaskExecutor::join (this=0x55b630ea1340) at src/mongo/executor/thread_pool_task_executor.cpp:173 #6 0x000055b62b3be251 in mongo::repl::ReplicationCoordinatorExternalStateImpl::shutdown (this=0x55b63002b600, opCtx=0x55b630eee140) at src/mongo/db/repl/replication_coordinator_external_state_impl.cpp:335 #7 0x000055b62b778efe in mongo::repl::ReplicationCoordinatorImpl::shutdown (this=0x55b6304c0000, opCtx=0x55b630eee140) at src/mongo/db/repl/replication_coordinator_impl.cpp:824 #8 0x000055b62aea917a in mongo::(anonymous namespace)::shutdownTask () at src/mongo/db/db.cpp:1281 #9 0x000055b62aeb97fb in std::_Function_handler::_M_invoke(std::_Any_data const&) (__functor=...) at /usr/include/c++/5/functional:1871 #10 0x000055b62b13ca24 in std::function::operator()() const (this=0x55b63006c400) at /usr/include/c++/5/functional:2267 #11 0x000055b62cd8325f in mongo::(anonymous namespace)::runTasks(std::stack, std::deque, std::allocator > > >) (tasks=std::stack wrapping: std::deque with 1 elements = {...}) at src/mongo/util/exit.cpp:61 #12 0x000055b62cd836f9 in mongo::shutdown (code=mongo::EXIT_CLEAN) at src/mongo/util/exit.cpp:140 #13 0x000055b62aead7c3 in mongo::exitCleanly (code=mongo::EXIT_CLEAN) at src/mongo/util/exit.h:81 #14 0x000055b62b835906 in mongo::(anonymous namespace)::signalProcessingThread (rotate=mongo::LogFileStatus::kNeedToRotateLogFile) at src/mongo/util/signal_handlers.cpp:197 #15 0x000055b62b836dc4 in std::_Bind_simple::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x55b6303066e8) at /usr/include/c++/5/functional:1531 #16 0x000055b62b836cd0 in std::_Bind_simple::operator()() (this=0x55b6303066e8) at /usr/include/c++/5/functional:1520 #17 0x000055b62b836c60 in std::thread::_Impl >::_M_run() (this=0x55b6303066d0) at /usr/include/c++/5/thread:115 #18 0x00007fac5353bc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #19 0x00007fac52d4e6ba in start_thread (arg=0x7fac5179c700) at pthread_create.c:333 #20 0x00007fac52a8441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 1: "mongod" (Thread 0x7fac54b09c80 (LWP 32216)) ---Type to continue, or q to quit--- #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fac5353691c in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x000055b62cd839b3 in std::condition_variable::wait >(std::unique_lock &, mongo::) (this=0x55b62e6bd760 , __lock=..., __p=...) at /usr/include/c++/5/condition_variable:98 #3 0x000055b62cd833c3 in mongo::waitForShutdown () at src/mongo/util/exit.cpp:93 #4 0x000055b62aea6fd6 in mongo::(anonymous namespace)::_initAndListen (listenPort=20000) at src/mongo/db/db.cpp:1045 #5 0x000055b62aea76e3 in mongo::(anonymous namespace)::initAndListen (listenPort=20000) at src/mongo/db/db.cpp:1050 #6 0x000055b62aea96a1 in mongo::mongoDbMain (argc=26, argv=0x7fffa9d5aed8, envp=0x7fffa9d5afb0) at src/mongo/db/db.cpp:1399 #7 0x000055b62ae8d388 in main (argc=26, argv=0x7fffa9d5aed8, envp=0x7fffa9d5afb0) at src/mongo/db/dbmain.cpp:49