diff --git a/src/mongo/db/concurrency/SConscript b/src/mongo/db/concurrency/SConscript index 90b0e21ba07..19d271dea91 100644 --- a/src/mongo/db/concurrency/SConscript +++ b/src/mongo/db/concurrency/SConscript @@ -57,6 +57,7 @@ env.Library( 'resource_catalog.cpp', ], LIBDEPS=[ + '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/service_context', '$BUILD_DIR/mongo/db/storage/storage_engine_parameters', '$BUILD_DIR/mongo/util/background_job', diff --git a/src/mongo/db/concurrency/lock_state.cpp b/src/mongo/db/concurrency/lock_state.cpp index f884e9eefd5..d048fcdcb40 100644 --- a/src/mongo/db/concurrency/lock_state.cpp +++ b/src/mongo/db/concurrency/lock_state.cpp @@ -568,6 +568,27 @@ void LockerImpl::lock(OperationContext* opCtx, ResourceId resId, LockMode mode, // `lockGlobal` must be called to lock `resourceIdGlobal`. invariant(resId != resourceIdGlobal); + if (shouldCheckForRSTLDeadlockChecks() && mode == MODE_X && + (resId.getType() == RESOURCE_DATABASE || resId.getType() == RESOURCE_COLLECTION) && opCtx && + opCtx->getServiceContext()->hasStartedUp() && + resId != resourceIdReplicationStateTransitionLock) { + auto replCoord = repl::ReplicationCoordinator::get(opCtx); + // Exclude this check if the RSTL lock is in X mode. Safe to do so because lock ordering + // guarantees this lock is taken before other locks if necessary for the operation. + if (auto it = _requests.find(resourceIdReplicationStateTransitionLock); it && + it->mode != MODE_X && replCoord->isReplEnabled() && + replCoord->canAcceptWritesForDatabase(opCtx, NamespaceString::kAdminDb)) { + if (auto client = opCtx->getClient()) { + if (client->isFromSystemConnection()) { + stdx::lock_guard lk(*client); + invariant(client->canKillSystemOperationInStepdown(lk), + "Acquiring a strong MODE_X lock in an unkillable operation while " + "holding the RSTL lock is forbidden"); + } + } + } + } + const LockResult result = _lockBegin(opCtx, resId, mode); // Fast, uncontended path diff --git a/src/mongo/db/concurrency/lock_state.h b/src/mongo/db/concurrency/lock_state.h index 1395567d4ce..0bc043f0c24 100644 --- a/src/mongo/db/concurrency/lock_state.h +++ b/src/mongo/db/concurrency/lock_state.h @@ -421,6 +421,22 @@ public: } }; +class SkipRSTLDeadlockChecks { +public: + explicit SkipRSTLDeadlockChecks(OperationContext* opCtx) : _opCtx(opCtx) { + auto lockState = opCtx->lockState(); + _originalValue = lockState->shouldCheckForRSTLDeadlockChecks(); + lockState->setCheckForRSTLDeadlock(false); + } + ~SkipRSTLDeadlockChecks() { + _opCtx->lockState()->setCheckForRSTLDeadlock(_originalValue); + } + +private: + OperationContext* _opCtx; + bool _originalValue; +}; + /** * RAII-style class to set the priority for the ticket admission mechanism when acquiring a global * lock. diff --git a/src/mongo/db/concurrency/locker.h b/src/mongo/db/concurrency/locker.h index 02ed66eaf52..704ea52a1aa 100644 --- a/src/mongo/db/concurrency/locker.h +++ b/src/mongo/db/concurrency/locker.h @@ -558,6 +558,14 @@ public: _debugInfo = info; } + bool shouldCheckForRSTLDeadlockChecks() const { + return _checkForRstlDeadlocks; + } + + void setCheckForRSTLDeadlock(bool newValue) { + _checkForRstlDeadlocks = newValue; + } + protected: Locker() {} @@ -584,6 +592,9 @@ protected: // Keeps state and statistics related to admission control. AdmissionContext _admCtx; + // Whether the lock method should check to avoid RSTL deadlocks. See + bool _checkForRstlDeadlocks = true; + private: bool _shouldConflictWithSecondaryBatchApplication = true; bool _shouldConflictWithSetFeatureCompatibilityVersion = true; diff --git a/src/mongo/db/index_builds_coordinator_mongod.cpp b/src/mongo/db/index_builds_coordinator_mongod.cpp index 70d0965d490..4bdd6d5e82a 100644 --- a/src/mongo/db/index_builds_coordinator_mongod.cpp +++ b/src/mongo/db/index_builds_coordinator_mongod.cpp @@ -37,6 +37,7 @@ #include "mongo/db/audit.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/index_build_entry_gen.h" +#include "mongo/db/concurrency/lock_state.h" #include "mongo/db/concurrency/locker.h" #include "mongo/db/concurrency/replication_state_transition_lock_guard.h" #include "mongo/db/curop.h" diff --git a/src/mongo/db/repl/bgsync.cpp b/src/mongo/db/repl/bgsync.cpp index 340aca59e80..ad04a609581 100644 --- a/src/mongo/db/repl/bgsync.cpp +++ b/src/mongo/db/repl/bgsync.cpp @@ -38,6 +38,7 @@ #include "mongo/db/auth/authorization_session.h" #include "mongo/db/client.h" #include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/lock_state.h" #include "mongo/db/concurrency/replication_state_transition_lock_guard.h" #include "mongo/db/dbhelpers.h" #include "mongo/db/index_builds_coordinator.h" diff --git a/src/mongo/db/repl/collection_cloner.cpp b/src/mongo/db/repl/collection_cloner.cpp index 2980833ef86..8d18685a5c4 100644 --- a/src/mongo/db/repl/collection_cloner.cpp +++ b/src/mongo/db/repl/collection_cloner.cpp @@ -34,6 +34,7 @@ #include "mongo/db/catalog/clustered_collection_options_gen.h" #include "mongo/db/catalog/clustered_collection_util.h" #include "mongo/db/commands/list_collections_filter.h" +#include "mongo/db/concurrency/lock_state.h" #include "mongo/db/index_build_entry_helpers.h" #include "mongo/db/index_builds_coordinator.h" #include "mongo/db/repl/collection_bulk_loader.h" diff --git a/src/mongo/db/repl/initial_syncer.cpp b/src/mongo/db/repl/initial_syncer.cpp index cd562d19af5..7ccca5a047f 100644 --- a/src/mongo/db/repl/initial_syncer.cpp +++ b/src/mongo/db/repl/initial_syncer.cpp @@ -44,6 +44,7 @@ #include "mongo/client/remote_command_retry_scheduler.h" #include "mongo/db/commands/server_status_metric.h" #include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/concurrency/lock_state.h" #include "mongo/db/feature_compatibility_version_parser.h" #include "mongo/db/index_builds_coordinator.h" #include "mongo/db/jsobj.h" diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp index 33720bec01d..af8250af8c8 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp @@ -585,6 +585,7 @@ Status ReplicationCoordinatorExternalStateImpl::storeLocalConfigDocument(Operati try { writeConflictRetry(opCtx, "save replica set config", kConfigCollectionNS.ns(), [&] { { + SkipRSTLDeadlockChecks guard(opCtx); // Writes to 'local.system.replset' must be untimestamped. WriteUnitOfWork wuow(opCtx); Lock::DBLock dbWriteLock(opCtx, kConfigDatabaseName, MODE_X); @@ -615,6 +616,7 @@ Status ReplicationCoordinatorExternalStateImpl::storeLocalConfigDocument(Operati Status ReplicationCoordinatorExternalStateImpl::replaceLocalConfigDocument( OperationContext* opCtx, const BSONObj& config) try { writeConflictRetry(opCtx, "replace replica set config", kConfigCollectionNS.ns(), [&] { + SkipRSTLDeadlockChecks guard(opCtx); WriteUnitOfWork wuow(opCtx); Lock::DBLock dbWriteLock(opCtx, kConfigDatabaseName, MODE_X); Helpers::emptyCollection(opCtx, kConfigCollectionNS); @@ -628,6 +630,7 @@ Status ReplicationCoordinatorExternalStateImpl::replaceLocalConfigDocument( Status ReplicationCoordinatorExternalStateImpl::createLocalLastVoteCollection( OperationContext* opCtx) { + SkipRSTLDeadlockChecks guard(opCtx); auto status = _storageInterface->createCollection( opCtx, NamespaceString::kLastVoteNamespace, CollectionOptions()); if (!status.isOK() && status.code() != ErrorCodes::NamespaceExists) { diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp index e6af5c08838..eaf0dc2257d 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl.cpp @@ -836,9 +836,11 @@ void ReplicationCoordinatorImpl::_initialSyncerCompletionFunction( _topCoord->resetMaintenanceCount(); } - ReplicaSetAwareServiceRegistry::get(_service).onInitialDataAvailable( - cc().makeOperationContext().get(), false /* isMajorityDataAvailable */); - + { + auto opCtx = cc().makeOperationContext(); + ReplicaSetAwareServiceRegistry::get(_service).onInitialDataAvailable( + opCtx.get(), false /* isMajorityDataAvailable */); + } // Transition from STARTUP2 to RECOVERING and start the producer and the applier. // If the member state is REMOVED, this will do nothing until we receive a config with // ourself in it. diff --git a/src/mongo/db/repl/tenant_file_importer_service.cpp b/src/mongo/db/repl/tenant_file_importer_service.cpp index 00e472fedb3..7c0b13b1618 100644 --- a/src/mongo/db/repl/tenant_file_importer_service.cpp +++ b/src/mongo/db/repl/tenant_file_importer_service.cpp @@ -37,6 +37,7 @@ #include "mongo/db/catalog_raii.h" #include "mongo/db/commands/tenant_migration_recipient_cmds_gen.h" #include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/concurrency/lock_state.h" #include "mongo/db/repl/oplog_applier.h" #include "mongo/db/repl/replication_auth.h" #include "mongo/db/repl/replication_coordinator.h" @@ -207,6 +208,8 @@ void TenantFileImporterService::interruptAll() { void TenantFileImporterService::_handleEvents(const UUID& migrationId) { auto opCtx = cc().makeOperationContext(); + SkipRSTLDeadlockChecks guard(opCtx.get()); + ON_BLOCK_EXIT([this, opId = opCtx->getOpID()] { stdx::lock_guard lk(_mutex); if (_opCtx && _opCtx->getOpID() == opId) { diff --git a/src/mongo/db/s/shard_server_catalog_cache_loader.cpp b/src/mongo/db/s/shard_server_catalog_cache_loader.cpp index 6264ee66cde..7c24014f8bd 100644 --- a/src/mongo/db/s/shard_server_catalog_cache_loader.cpp +++ b/src/mongo/db/s/shard_server_catalog_cache_loader.cpp @@ -33,6 +33,7 @@ #include "mongo/db/catalog/rename_collection.h" #include "mongo/db/client.h" +#include "mongo/db/concurrency/lock_state.h" #include "mongo/db/db_raii.h" #include "mongo/db/operation_context.h" #include "mongo/db/operation_context_group.h" @@ -1168,6 +1169,7 @@ void ShardServerCatalogCacheLoader::_runDbTasks(StringData dbName) { void ShardServerCatalogCacheLoader::_updatePersistedCollAndChunksMetadata( OperationContext* opCtx, const NamespaceString& nss) { + SkipRSTLDeadlockChecks guard(opCtx); stdx::unique_lock lock(_mutex); const CollAndChunkTask& task = _collAndChunkTaskLists[nss].front(); diff --git a/src/mongo/db/s/sharding_initialization_mongod.cpp b/src/mongo/db/s/sharding_initialization_mongod.cpp index 26a093ab88d..40f619141d1 100644 --- a/src/mongo/db/s/sharding_initialization_mongod.cpp +++ b/src/mongo/db/s/sharding_initialization_mongod.cpp @@ -41,6 +41,7 @@ #include "mongo/db/catalog_shard_feature_flag_gen.h" #include "mongo/db/client_metadata_propagation_egress_hook.h" #include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/concurrency/lock_state.h" #include "mongo/db/dbhelpers.h" #include "mongo/db/keys_collection_client_direct.h" #include "mongo/db/keys_collection_client_sharded.h" @@ -488,6 +489,7 @@ void ShardingInitializationMongoD::updateShardIdentityConfigString( write_ops::UpdateModification::parseFromClassicUpdate(updateObj)); try { + SkipRSTLDeadlockChecks guard(opCtx); AutoGetDb autoDb(opCtx, NamespaceString::kServerConfigurationNamespace.dbName(), MODE_X); auto result = update(opCtx, autoDb.ensureDbExists(opCtx), updateReq); diff --git a/src/mongo/db/service_context.h b/src/mongo/db/service_context.h index 6ef2e1dc145..b40309d945b 100644 --- a/src/mongo/db/service_context.h +++ b/src/mongo/db/service_context.h @@ -627,6 +627,10 @@ public: return _userWritesAllowed.load(); } + bool hasStartedUp() const { + return _startupComplete; + } + LockedClient getLockedClient(OperationId id); private: