I can only seem to reproduce this particular crash with legacy config servers.
==8420== ERROR: AddressSanitizer: heap-use-after-free on address 0x600600615b80 at pc 0x147add4 bp 0x7f9a8a0440a0 sp 0x7f9a8a044098 READ of size 8 at 0x600600615b80 thread T49 #0 0x147add3 in mongo::ShardRegistry::runCommandWithNotMasterRetries(std::string const&, std::string const&, mongo::BSONObj const&, mongo::BSONObj const&) /home/s/code/mongo/mongo/src/mongo/s/client/shard_registry.cpp:400 #1 0x147ab2a in mongo::ShardRegistry::runCommandWithNotMasterRetries(std::string const&, std::string const&, mongo::BSONObj const&) /home/s/code/mongo/mongo/src/mongo/s/client/shard_registry.cpp:382 #2 0x13867f1 in mongo::CatalogManager::dropCollection(mongo::OperationContext*, mongo::NamespaceString const&) /home/s/code/mongo/mongo/src/mongo/s/catalog/catalog_manager.cpp:735 #3 0x14d6921 in mongo::dbgrid_pub_cmds::DropCmd::run(mongo::OperationContext*, std::string const&, mongo::BSONObj&, int, std::string&, mongo::BSONObjBuilder&) /home/s/code/mongo/mongo/src/mongo/s/commands/commands_public.cpp:448 #4 0x152969e in mongo::Command::execCommandClientBasic(mongo::OperationContext*, mongo::Command*, mongo::ClientBasic&, int, char const*, mongo::BSONObj&, mongo::BSONObjBuilder&) /home/s/code/mongo/mongo/src/mongo/s/s_only.cpp:128 #5 0x1529d99 in mongo::Command::runAgainstRegistered(char const*, mongo::BSONObj&, mongo::BSONObjBuilder&, int) /home/s/code/mongo/mongo/src/mongo/s/s_only.cpp:169 #6 0x153c831 in mongo::Strategy::clientCommandOp(mongo::Request&) /home/s/code/mongo/mongo/src/mongo/s/strategy.cpp:370 #7 0x15282e4 in mongo::Request::process(int) /home/s/code/mongo/mongo/src/mongo/s/request.cpp:111 #8 0xdf5f95 in mongo::ShardedMessageHandler::process(mongo::Message&, mongo::AbstractMessagingPort*) /home/s/code/mongo/mongo/src/mongo/s/server.cpp:135 #9 0x15db5d1 in mongo::PortMessageServer::handleIncomingMsg(void*) /home/s/code/mongo/mongo/src/mongo/util/net/message_server_port.cpp:229 #10 0x7f9a94988b97 (/usr/lib/x86_64-linux-gnu/libasan.so.0+0x18b97) #11 0x7f9a93b2e181 in start_thread /build/buildd/eglibc-2.19/nptl/pthread_create.c:312 #12 0x7f9a9385b47c in clone /build/buildd/eglibc-2.19/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:111 0x600600615b80 is located 0 bytes inside of 24-byte region [0x600600615b80,0x600600615b98) freed by thread T49 here: #0 0x7f9a949819da in operator delete(void*) (/usr/lib/x86_64-linux-gnu/libasan.so.0+0x119da) #1 0xf04795 in mongo::RemoteCommandTargeterStandalone::~RemoteCommandTargeterStandalone() /home/s/code/mongo/mongo/src/mongo/client/remote_command_targeter_standalone.h:40 #2 0x146f5a4 in std::default_delete<mongo::RemoteCommandTargeter>::operator()(mongo::RemoteCommandTargeter*) const /usr/include/c++/4.8/bits/unique_ptr.h:67 #3 0x146f435 in std::unique_ptr<mongo::RemoteCommandTargeter, std::default_delete<mongo::RemoteCommandTargeter> >::~unique_ptr() /usr/include/c++/4.8/bits/unique_ptr.h:184 #4 0x146f053 in mongo::Shard::~Shard() /home/s/code/mongo/mongo/src/mongo/s/client/shard.h:50 #5 0x14829e1 in void __gnu_cxx::new_allocator<mongo::Shard>::destroy<mongo::Shard>(mongo::Shard*) /usr/include/c++/4.8/ext/new_allocator.h:124 #6 0x148299d in std::enable_if<std::allocator_traits<std::allocator<mongo::Shard> >::__destroy_helper<mongo::Shard>::value, void>::type std::allocator_traits<std::allocator<mongo::Shard> >::_S_destroy<mongo::Shard>(std::allocator<mongo::Shard>&, mongo::Shard*) /usr/include/c++/4.8/bits/alloc_traits.h:281 #7 0x1482953 in void std::allocator_traits<std::allocator<mongo::Shard> >::destroy<mongo::Shard>(std::allocator<mongo::Shard>&, mongo::Shard*) /usr/include/c++/4.8/bits/alloc_traits.h:405 #8 0x148284d in std::_Sp_counted_ptr_inplace<mongo::Shard, std::allocator<mongo::Shard>, (__gnu_cxx::_Lock_policy)2>::_M_dispose() /usr/include/c++/4.8/bits/shared_ptr_base.h:407 #9 0xdf9f4a in std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release() /usr/include/c++/4.8/bits/shared_ptr_base.h:144 #10 0xdf7947 in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count() /usr/include/c++/4.8/bits/shared_ptr_base.h:546 #11 0xee80fb in std::__shared_ptr<mongo::Shard, (__gnu_cxx::_Lock_policy)2>::~__shared_ptr() /usr/include/c++/4.8/bits/shared_ptr_base.h:781 #12 0xee812f in std::shared_ptr<mongo::Shard>::~shared_ptr() /usr/include/c++/4.8/bits/shared_ptr.h:93 #13 0x147ad67 in mongo::ShardRegistry::runCommandWithNotMasterRetries(std::string const&, std::string const&, mongo::BSONObj const&, mongo::BSONObj const&) /home/s/code/mongo/mongo/src/mongo/s/client/shard_registry.cpp:396 #14 0x147ab2a in mongo::ShardRegistry::runCommandWithNotMasterRetries(std::string const&, std::string const&, mongo::BSONObj const&) /home/s/code/mongo/mongo/src/mongo/s/client/shard_registry.cpp:382 #15 0x13867f1 in mongo::CatalogManager::dropCollection(mongo::OperationContext*, mongo::NamespaceString const&) /home/s/code/mongo/mongo/src/mongo/s/catalog/catalog_manager.cpp:735 #16 0x14d6921 in mongo::dbgrid_pub_cmds::DropCmd::run(mongo::OperationContext*, std::string const&, mongo::BSONObj&, int, std::string&, mongo::BSONObjBuilder&) /home/s/code/mongo/mongo/src/mongo/s/commands/commands_public.cpp:448 #17 0x152969e in mongo::Command::execCommandClientBasic(mongo::OperationContext*, mongo::Command*, mongo::ClientBasic&, int, char const*, mongo::BSONObj&, mongo::BSONObjBuilder&) /home/s/code/mongo/mongo/src/mongo/s/s_only.cpp:128 #18 0x1529d99 in mongo::Command::runAgainstRegistered(char const*, mongo::BSONObj&, mongo::BSONObjBuilder&, int) /home/s/code/mongo/mongo/src/mongo/s/s_only.cpp:169 #19 0x153c831 in mongo::Strategy::clientCommandOp(mongo::Request&) /home/s/code/mongo/mongo/src/mongo/s/strategy.cpp:370 #20 0x15282e4 in mongo::Request::process(int) /home/s/code/mongo/mongo/src/mongo/s/request.cpp:111 #21 0xdf5f95 in mongo::ShardedMessageHandler::process(mongo::Message&, mongo::AbstractMessagingPort*) /home/s/code/mongo/mongo/src/mongo/s/server.cpp:135 #22 0x15db5d1 in mongo::PortMessageServer::handleIncomingMsg(void*) /home/s/code/mongo/mongo/src/mongo/util/net/message_server_port.cpp:229 #23 0x7f9a94988b97 (/usr/lib/x86_64-linux-gnu/libasan.so.0+0x18b97) previously allocated by thread T46 here: #0 0x7f9a9498181a in operator new(unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.0+0x1181a) #1 0xf03ded in boost::detail::up_if_not_array<mongo::RemoteCommandTargeterStandalone>::type boost::make_unique<mongo::RemoteCommandTargeterStandalone, mongo::HostAndPort const&>(mongo::HostAndPort const&) /home/s/code/mongo/mongo/src/third_party/boost-1.56.0/boost/smart_ptr/make_unique_object.hpp:28 #2 0xf03bf9 in mongo::RemoteCommandTargeterFactoryImpl::create(mongo::ConnectionString const&) /home/s/code/mongo/mongo/src/mongo/client/remote_command_targeter_factory_impl.cpp:52 #3 0x147904f in mongo::ShardRegistry::_addShard_inlock(mongo::ShardType const&) /home/s/code/mongo/mongo/src/mongo/s/client/shard_registry.cpp:237 #4 0x1477f8e in mongo::ShardRegistry::reload() /home/s/code/mongo/mongo/src/mongo/s/client/shard_registry.cpp:121 #5 0x147812e in mongo::ShardRegistry::getShard(std::string const&) /home/s/code/mongo/mongo/src/mongo/s/client/shard_registry.cpp:132 #6 0x15428e3 in mongo::(anonymous namespace)::initShardVersionEmptyNS(mongo::DBClientBase*) /home/s/code/mongo/mongo/src/mongo/s/version_manager.cpp:208 #7 0x15431ea in mongo::(anonymous namespace)::checkShardVersion(mongo::DBClientBase*, std::string const&, std::shared_ptr<mongo::ChunkManager>, bool, int) /home/s/code/mongo/mongo/src/mongo/s/version_manager.cpp:285 #8 0x15458f8 in mongo::VersionManager::checkShardVersionCB(mongo::ShardConnection*, bool, int) /home/s/code/mongo/mongo/src/mongo/s/version_manager.cpp:483 #9 0x14720ff in mongo::ShardConnection::_finishInit() /home/s/code/mongo/mongo/src/mongo/s/client/shard_connection.cpp:453 #10 0x1476ff1 in mongo::ShardConnection::get() /home/s/code/mongo/mongo/src/mongo/s/client/shard_connection.h:63 #11 0x145b244 in mongo::DBClientMultiCommand::sendAll() /home/s/code/mongo/mongo/src/mongo/s/client/dbclient_multi_command.cpp:162 #12 0x13b5501 in mongo::ConfigCoordinator::_checkConfigString(mongo::BatchedCommandResponse*) /home/s/code/mongo/mongo/src/mongo/s/catalog/legacy/config_coordinator.cpp:316 #13 0x13b6180 in mongo::ConfigCoordinator::executeBatch(mongo::BatchedCommandRequest const&, mongo::BatchedCommandResponse*) /home/s/code/mongo/mongo/src/mongo/s/catalog/legacy/config_coordinator.cpp:417 #14 0x13a1c8c in mongo::CatalogManagerLegacy::writeConfigServerDirect(mongo::BatchedCommandRequest const&, mongo::BatchedCommandResponse*) /home/s/code/mongo/mongo/src/mongo/s/catalog/legacy/catalog_manager_legacy.cpp:972 #15 0x1382d25 in mongo::CatalogManager::insert(std::string const&, mongo::BSONObj const&, mongo::BatchedCommandResponse*) /home/s/code/mongo/mongo/src/mongo/s/catalog/catalog_manager.cpp:427 #16 0x139c366 in mongo::CatalogManagerLegacy::logChange(std::string const&, std::string const&, std::string const&, mongo::BSONObj const&) /home/s/code/mongo/mongo/src/mongo/s/catalog/legacy/catalog_manager_legacy.cpp:599 #17 0x13857b4 in mongo::CatalogManager::dropCollection(mongo::OperationContext*, mongo::NamespaceString const&) /home/s/code/mongo/mongo/src/mongo/s/catalog/catalog_manager.cpp:647 #18 0x14d6921 in mongo::dbgrid_pub_cmds::DropCmd::run(mongo::OperationContext*, std::string const&, mongo::BSONObj&, int, std::string&, mongo::BSONObjBuilder&) /home/s/code/mongo/mongo/src/mongo/s/commands/commands_public.cpp:448 #19 0x152969e in mongo::Command::execCommandClientBasic(mongo::OperationContext*, mongo::Command*, mongo::ClientBasic&, int, char const*, mongo::BSONObj&, mongo::BSONObjBuilder&) /home/s/code/mongo/mongo/src/mongo/s/s_only.cpp:128 #20 0x1529d99 in mongo::Command::runAgainstRegistered(char const*, mongo::BSONObj&, mongo::BSONObjBuilder&, int) /home/s/code/mongo/mongo/src/mongo/s/s_only.cpp:169 #21 0x153c831 in mongo::Strategy::clientCommandOp(mongo::Request&) /home/s/code/mongo/mongo/src/mongo/s/strategy.cpp:370 #22 0x15282e4 in mongo::Request::process(int) /home/s/code/mongo/mongo/src/mongo/s/request.cpp:111 #23 0xdf5f95 in mongo::ShardedMessageHandler::process(mongo::Message&, mongo::AbstractMessagingPort*) /home/s/code/mongo/mongo/src/mongo/s/server.cpp:135 #24 0x15db5d1 in mongo::PortMessageServer::handleIncomingMsg(void*) /home/s/code/mongo/mongo/src/mongo/util/net/message_server_port.cpp:229 #25 0x7f9a94988b97 (/usr/lib/x86_64-linux-gnu/libasan.so.0+0x18b97) Thread T49 created by T0 here: #0 0x7f9a9497ab5b in __interceptor_pthread_create (/usr/lib/x86_64-linux-gnu/libasan.so.0+0xab5b) #1 0x15dacc1 in mongo::PortMessageServer::accepted(std::shared_ptr<mongo::Socket>, long long) /home/s/code/mongo/mongo/src/mongo/util/net/message_server_port.cpp:148 #2 0x15d1080 in mongo::Listener::initAndListen() /home/s/code/mongo/mongo/src/mongo/util/net/listen.cpp:351 #3 0x15dafe1 in mongo::PortMessageServer::run() /home/s/code/mongo/mongo/src/mongo/util/net/message_server_port.cpp:176 #4 0xdf1943 in mongo::start(mongo::MessageServer::Options const&) /home/s/code/mongo/mongo/src/mongo/s/server.cpp:180 #5 0xdf2192 in runMongosServer(bool) /home/s/code/mongo/mongo/src/mongo/s/server.cpp:266 #6 0xdf254f in _main() /home/s/code/mongo/mongo/src/mongo/s/server.cpp:324 #7 0xdf2993 in mongoSMain(int, char**, char**) /home/s/code/mongo/mongo/src/mongo/s/server.cpp:395 #8 0xdf2dc4 in main /home/s/code/mongo/mongo/src/mongo/s/server.cpp:423 #9 0x7f9a93782ec4 in __libc_start_main /build/buildd/eglibc-2.19/csu/libc-start.c:287 Thread T46 created by T0 here: #0 0x7f9a9497ab5b in __interceptor_pthread_create (/usr/lib/x86_64-linux-gnu/libasan.so.0+0xab5b) #1 0x15dacc1 in mongo::PortMessageServer::accepted(std::shared_ptr<mongo::Socket>, long long) /home/s/code/mongo/mongo/src/mongo/util/net/message_server_port.cpp:148 #2 0x15d1080 in mongo::Listener::initAndListen() /home/s/code/mongo/mongo/src/mongo/util/net/listen.cpp:351 #3 0x15dafe1 in mongo::PortMessageServer::run() /home/s/code/mongo/mongo/src/mongo/util/net/message_server_port.cpp:176 #4 0xdf1943 in mongo::start(mongo::MessageServer::Options const&) /home/s/code/mongo/mongo/src/mongo/s/server.cpp:180 #5 0xdf2192 in runMongosServer(bool) /home/s/code/mongo/mongo/src/mongo/s/server.cpp:266 #6 0xdf254f in _main() /home/s/code/mongo/mongo/src/mongo/s/server.cpp:324 #7 0xdf2993 in mongoSMain(int, char**, char**) /home/s/code/mongo/mongo/src/mongo/s/server.cpp:395 #8 0xdf2dc4 in main /home/s/code/mongo/mongo/src/mongo/s/server.cpp:423 #9 0x7f9a93782ec4 in __libc_start_main /build/buildd/eglibc-2.19/csu/libc-start.c:287 SUMMARY: AddressSanitizer: heap-use-after-free /home/s/code/mongo/mongo/src/mongo/s/client/shard_registry.cpp:400 mongo::ShardRegistry::runCommandWithNotMasterRetries(std::string const&, std::string const&, mongo::BSONObj const&, mongo::BSONObj const&) Shadow bytes around the buggy address: 0x0c01400bab20: fa fa fd fd fd fd fa fa fd fd fd fd fa fa fd fd 0x0c01400bab30: fd fd fa fa fd fd fd fd fa fa fd fd fd fd fa fa 0x0c01400bab40: fd fd fd fd fa fa fd fd fd fa fa fa fd fd fd fd 0x0c01400bab50: fa fa fd fd fd fa fa fa fd fd fd fa fa fa fd fd 0x0c01400bab60: fd fd fa fa fd fd fd fa fa fa fd fd fd fa fa fa =>0x0c01400bab70:[fd]fd fd fa fa fa fd fd fd fd fa fa fd fd fd fa 0x0c01400bab80: fa fa fd fd fd fd fa fa fd fd fd fd fa fa fd fd 0x0c01400bab90: fd fd fa fa fd fd fd fd fa fa fd fd fd fd fa fa 0x0c01400baba0: fd fd fd fd fa fa fd fd fd fa fa fa fd fd fd fd 0x0c01400babb0: fa fa fd fd fd fa fa fa fd fd fd fa fa fa fd fd 0x0c01400babc0: fd fd fa fa fd fd fd fa fa fa fd fd fd fa fa fa Shadow byte legend (one shadow byte represents 8 application bytes): Addressable: 00 Partially addressable: 01 02 03 04 05 06 07 Heap left redzone: fa Heap righ redzone: fb Freed Heap region: fd Stack left redzone: f1 Stack mid redzone: f2 Stack right redzone: f3 Stack partial redzone: f4 Stack after return: f5 Stack use after scope: f8 Global redzone: f9 Global init order: f6 Poisoned by user: f7 ASan internal: fe
Version: c54e23ccee372703cb2dc714762f9beaf4ad0e10
- related to
-
SERVER-19929 Audit sharding code for potential use-after-frees
- Closed