Critical - P2 during test scalability of mongos/shard, found performance regression of mongos due to lock sleep
the traffic is YCSB
- single doc insert
- shard with {_id: hashed}
- total 7 shards, each is 3 member replica set
- writer with 64 threads
- only change mongos binary, 3.1.1/3.1.2 show lower throughput.
3.0.2 3.1.2 throughput 43451 7065
from off-cpu perf analysis
## with master 3.1.2 (c8e2c0546b30621f78cd436d96714cc064bbb8a7)
-36.26%-- futex_wait_queue_me
futex_wait
do_futex
sys_futex
system_call_fastpath
|
|--100.00%-- __lll_lock_wait
| |
| |--85.34%-- mongo::StaticShardInfo::reload()
| | |
| | |--100.00%-- mongo::Shard::findIfExists(std::string const&)
| | | mongo::DBClientShardResolver::chooseWriteHost(std::string const&, mongo::ConnectionString*) const
| | | mongo::BatchWriteExec::executeBatch(mongo::BatchedCommandRequest const&, mongo::BatchedCommandResponse*
| | | mongo::ClusterWriter::write(mongo::BatchedCommandRequest const&, mongo::BatchedCommandResponse*)
| | | |
| | | --100.00%-- mongo::(anonymous namespace)::ClusterWriteCmd::run(mongo::OperationContext*, std::string c
| | | mongo::Command::execCommandClientBasic(mongo::OperationContext*, mongo::Command*, mongo::Cli
| | | mongo::Command::runAgainstRegistered(char const*, mongo::BSONObj&, mongo::BSONObjBuilder&, i
| | | mongo::Strategy::clientCommandOp(mongo::Request&)
| | | mongo::Request::process(int)
| | | mongo::ShardedMessageHandler::process(mongo::Message&, mongo::AbstractMessagingPort*, mongo:
| | | mongo::PortMessageServer::handleIncomingMsg(void*)
| | | start_thread
| | --0.00%-- [...]
| |
| |--14.42%-- mongo::Shard::findIfExists(std::string const&)
| | mongo::DBClientShardResolver::chooseWriteHost(std::string const&, mongo::ConnectionString*) const
| | mongo::BatchWriteExec::executeBatch(mongo::BatchedCommandRequest const&, mongo::BatchedCommandResponse*)
| | mongo::ClusterWriter::write(mongo::BatchedCommandRequest const&, mongo::BatchedCommandResponse*)
| | |
| | --100.00%-- mongo::(anonymous namespace)::ClusterWriteCmd::run(mongo::OperationContext*, std::string const&, mong
| | mongo::Command::execCommandClientBasic(mongo::OperationContext*, mongo::Command*, mongo::ClientBasic&,
| | mongo::Command::runAgainstRegistered(char const*, mongo::BSONObj&, mongo::BSONObjBuilder&, int)
| | mongo::Strategy::clientCommandOp(mongo::Request&)
| | mongo::Request::process(int)
| | mongo::ShardedMessageHandler::process(mongo::Message&, mongo::AbstractMessagingPort*, mongo::LastError*
| | mongo::PortMessageServer::handleIncomingMsg(void*)
| | start_thread
| --0.24%-- [...]
--0.00%-- [...]