• 9/18 splunk example
• Hundreds of repros in the past 7 days splunk

#0 0x00007f6c233b43ad in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f6c233adcfd in pthread_mutex_lock () from /lib64/libpthread.so.0

#2 0x000055df1aa39160 in mongo::latch_detail::Mutex::lock() ()

#3 0x000055df1646e6da in streams::MetricManager::registerCounter(std::_cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::vector<std::pair<std::cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::_cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >) ()

#4 0x000055df1645dabf in streams::SourceOperator::SourceOperator(streams::Context*, int) ()

#5 0x000055df164177b7 in streams::KafkaConsumerOperator::KafkaConsumerOperator(streams::Context*, streams::KafkaConsumerOperator::Options) ()

#6 0x000055df1643b565 in streams::OperatorFactory::toSourceOperator(streams::KafkaConsumerOperator::Options) ()

#7 0x000055df164441bb in streams::(anonymous namespace)::fromSourceSpec(mongo::BSONObj const&, streams::Context*, streams::OperatorFactory*, absl::lts_20211102::node_hash_map<std::_cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, mongo::Connection, absl::lts_20211102::container_internal::StringHash, absl::lts_20211102::container_internal::StringEq, std::allocator<std::pair<std::_cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, mongo::Connection> > > const&, bool) ()

#8 0x000055df16446d81 in streams::Parser::fromBson(std::vector<mongo::BSONObj, std::allocator<mongo::BSONObj> > const&) const ()

#9 0x000055df163e3a0d in streams::StreamManager::createStreamProcessorInfoLocked(mongo::StartStreamProcessorCommand const&) ()

#10 0x000055df163e902b in streams::StreamManager::startStreamProcessor(mongo::StartStreamProcessorCommand const&) ()
this thread has the stream mgr lock and is waiting on metric mgr lock
#0 0x00007f6c233b43ad in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f6c233adcfd in pthread_mutex_lock () from /lib64/libpthread.so.0

#2 0x000055df1aa39160 in mongo::latch_detail::Mutex::lock() ()

#3 0x000055df163dfcf3 in std::_Function_handler<double (), streams::StreamManager::StreamManager(mongo::ServiceContext*, streams::StreamManager::Options)::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()

#4 0x000055df163ea32a in streams::StreamManager::getMetrics() ()
this thread has the metric mgr lock and is waiting on stream manager lock