From 4221e438647009199a0a979e13a218aee90805dd Mon Sep 17 00:00:00 2001 From: Ben Judd Date: Fri, 6 Jul 2018 11:57:18 -0400 Subject: [PATCH] SERVER-1773 Created working truncate command, needs to be plugged into oplog --- jstests/auth/lib/commands_lib.js | 16 +++ jstests/core/truncate.js | 30 +++++ jstests/core/views/views_all_commands.js | 4 + .../set_read_and_write_concerns.js | 1 + jstests/noPassthrough/truncate.js | 20 +++ .../database_and_shard_versioning_all_commands.js | 10 ++ .../sharding/libs/last_stable_mongos_commands.js | 2 +- .../sharding/safe_secondary_reads_drop_recreate.js | 1 + ...eads_single_migration_suspend_range_deletion.js | 1 + ...condary_reads_single_migration_waitForDelete.js | 1 + src/mongo/db/catalog/collection_impl.cpp | 35 ++---- src/mongo/db/catalog/uuid_catalog.h | 3 + src/mongo/db/commands/SConscript | 24 +++- src/mongo/db/commands/truncate.cpp | 138 +++++++++++++++++++++ src/mongo/db/commands/truncate.h | 40 ++++++ src/mongo/db/free_mon/free_mon_op_observer.h | 4 + src/mongo/db/index/index_access_method.cpp | 4 + src/mongo/db/index/index_access_method.h | 6 + src/mongo/db/op_observer.h | 7 ++ src/mongo/db/op_observer_impl.cpp | 22 ++++ src/mongo/db/op_observer_impl.h | 3 + src/mongo/db/op_observer_noop.h | 3 + src/mongo/db/op_observer_registry.h | 7 ++ src/mongo/db/repl/SConscript | 1 + src/mongo/db/repl/oplog.cpp | 11 ++ src/mongo/db/repl/oplog_entry.cpp | 2 + src/mongo/db/repl/oplog_entry.h | 1 + src/mongo/db/repl/oplog_entry.idl | 7 ++ src/mongo/db/s/config_server_op_observer.h | 4 + src/mongo/db/s/shard_server_op_observer.h | 4 + src/mongo/db/storage/biggie/biggie_sorted_impl.cpp | 3 +- src/mongo/db/storage/biggie/biggie_sorted_impl.h | 2 +- src/mongo/db/storage/devnull/devnull_kv_engine.cpp | 2 + .../ephemeral_for_test_btree_impl.cpp | 22 ++++ src/mongo/db/storage/mobile/mobile_index.cpp | 6 + src/mongo/db/storage/mobile/mobile_index.h | 2 + src/mongo/db/storage/sorted_data_interface.h | 5 + .../db/storage/wiredtiger/wiredtiger_index.cpp | 12 ++ src/mongo/db/storage/wiredtiger/wiredtiger_index.h | 2 + src/mongo/embedded/capi_test.cpp | 1 + src/mongo/s/commands/SConscript | 1 + src/mongo/s/commands/cluster_truncate_cmd.cpp | 125 +++++++++++++++++++ 42 files changed, 558 insertions(+), 37 deletions(-) create mode 100644 jstests/core/truncate.js create mode 100644 jstests/noPassthrough/truncate.js create mode 100644 src/mongo/db/commands/truncate.cpp create mode 100644 src/mongo/db/commands/truncate.h create mode 100644 src/mongo/s/commands/cluster_truncate_cmd.cpp diff --git a/jstests/auth/lib/commands_lib.js b/jstests/auth/lib/commands_lib.js index 1ef33169fd..93b5d2de74 100644 --- a/jstests/auth/lib/commands_lib.js +++ b/jstests/auth/lib/commands_lib.js @@ -5510,6 +5510,22 @@ var authCommandsLib = { } ] }, + { + testname: "truncate", + command: {truncate: "test"}, + setup: function(db) { + db.test.insert({}); + }, + teardown: function(db) { + db.test.drop(); + }, + testcases: [{ + runOnDb: firstDbName, + // Doesn't work with "restore" + roles: {readWrite: 1, readWriteAnyDatabase: 1, dbOwner: 1, root: 1, __system: 1}, + privileges: [{resource: {db: firstDbName, collection: "test"}, actions: ["remove"]}], + }] + }, { testname: "unsetSharding", command: {unsetSharding: "x"}, diff --git a/jstests/core/truncate.js b/jstests/core/truncate.js new file mode 100644 index 0000000000..1ec410d81f --- /dev/null +++ b/jstests/core/truncate.js @@ -0,0 +1,30 @@ +(function() { + "use strict"; + var collName = "truncate"; + var coll = db.truncate; + + // Fastcount can't be used with suite replica_sets_kill_primary_jscore_passthrough + let assertEmpty = () => { + assert.eq(coll.find().itcount(), 0); + }; + // Basic test + coll.drop(); + coll.insert({}); + assert.eq(coll.find().itcount(), 1); + assert.commandWorked(coll.runCommand("truncate")); + assertEmpty(); + + // Test on empty + coll.drop(); + db.createCollection(collName); + assert.commandWorked(coll.runCommand("truncate")); + assertEmpty(); + + // Test on non-existant + coll.drop(); + assert.commandWorked(coll.runCommand("truncate")); + assertEmpty(); + + // Cleanup + coll.drop(); +})(); diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js index cd1db185ba..5d2403c084 100644 --- a/jstests/core/views/views_all_commands.js +++ b/jstests/core/views/views_all_commands.js @@ -519,6 +519,10 @@ command: {touch: "view", data: true}, expectFailure: true, }, + truncate: { + command: {truncate: "view"}, + expectFailure: true, + }, unsetSharding: {skip: isAnInternalCommand}, update: {command: {update: "view", updates: [{q: {x: 1}, u: {x: 2}}]}, expectFailure: true}, updateRole: { diff --git a/jstests/libs/override_methods/set_read_and_write_concerns.js b/jstests/libs/override_methods/set_read_and_write_concerns.js index b68d6c1c5a..51630cdc82 100644 --- a/jstests/libs/override_methods/set_read_and_write_concerns.js +++ b/jstests/libs/override_methods/set_read_and_write_concerns.js @@ -96,6 +96,7 @@ "revokeRolesFromRole", "revokeRolesFromUser", "setFeatureCompatibilityVersion", + "truncate", "update", "updateRole", "updateUser", diff --git a/jstests/noPassthrough/truncate.js b/jstests/noPassthrough/truncate.js new file mode 100644 index 0000000000..e665eaec67 --- /dev/null +++ b/jstests/noPassthrough/truncate.js @@ -0,0 +1,20 @@ +// Tests the truncate command on mongod with bad fcv +(function() { + 'use strict'; + load('jstests/libs/feature_compatibility_version.js'); + + var sourceMongodConn = MongoRunner.runMongod({}); + var testDB = sourceMongodConn.getDB('test'); + + // Ensure sourceMongodConn has featureCompatibilityVersion=4.0 which will fail with truncate + assert.commandWorked(sourceMongodConn.adminCommand({setFeatureCompatibilityVersion: "4.0"})); + + testDB.foo.insert({}); + assert.commandFailed(testDB.foo.runCommand("truncate")); + + // Should work with 4.2 + assert.commandWorked(sourceMongodConn.adminCommand({setFeatureCompatibilityVersion: "4.2"})); + assert.commandWorked(testDB.foo.runCommand("truncate")); + + MongoRunner.stopMongod(sourceMongodConn); +})(); diff --git a/jstests/sharding/database_and_shard_versioning_all_commands.js b/jstests/sharding/database_and_shard_versioning_all_commands.js index 6016478ebc..1243274b56 100644 --- a/jstests/sharding/database_and_shard_versioning_all_commands.js +++ b/jstests/sharding/database_and_shard_versioning_all_commands.js @@ -396,6 +396,16 @@ split: {skip: "does not forward command to primary shard"}, splitVector: {skip: "does not forward command to primary shard"}, startSession: {skip: "executes locally on mongos (not sent to any remote node)"}, + truncate: { + skipProfilerCheck: false, + sendsDbVersion: true, + sendsShardVersion: true, + setUp: function(mongosConn) { + // Expects the collection to exist, and doesn't implicitly create it. + assert.commandWorked(mongosConn.getDB(dbName).runCommand({create: collName})); + }, + command: {truncate: collName}, + }, update: { skipProfilerCheck: true, sendsDbVersion: false, diff --git a/jstests/sharding/libs/last_stable_mongos_commands.js b/jstests/sharding/libs/last_stable_mongos_commands.js index 0f3a5b7a19..86d71fcd6d 100644 --- a/jstests/sharding/libs/last_stable_mongos_commands.js +++ b/jstests/sharding/libs/last_stable_mongos_commands.js @@ -14,4 +14,4 @@ const commandsRemovedFromMongosIn42 = [ // These commands were added in mongos 4.2, so will not appear in the listCommands output of a 4.0 // mongos. We will allow these commands to have a test defined without always existing on the mongos // being used. -const commandsAddedToMongosIn42 = ['abortTransaction', 'commitTransaction']; +const commandsAddedToMongosIn42 = ['abortTransaction', 'commitTransaction', 'truncate']; diff --git a/jstests/sharding/safe_secondary_reads_drop_recreate.js b/jstests/sharding/safe_secondary_reads_drop_recreate.js index 1aaa2dbcb6..49ad67d63e 100644 --- a/jstests/sharding/safe_secondary_reads_drop_recreate.js +++ b/jstests/sharding/safe_secondary_reads_drop_recreate.js @@ -291,6 +291,7 @@ startSession: {skip: "does not return user data"}, top: {skip: "does not return user data"}, touch: {skip: "does not return user data"}, + truncate: {skip: "primary only"}, unsetSharding: {skip: "does not return user data"}, update: {skip: "primary only"}, updateRole: {skip: "primary only"}, diff --git a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js index fc702d4049..a4c0058cfc 100644 --- a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js +++ b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js @@ -326,6 +326,7 @@ startSession: {skip: "does not return user data"}, top: {skip: "does not return user data"}, touch: {skip: "does not return user data"}, + truncate: {skip: "primary only"}, unsetSharding: {skip: "does not return user data"}, update: {skip: "primary only"}, updateRole: {skip: "primary only"}, diff --git a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js index d449250cb3..a348f83ca7 100644 --- a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js +++ b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js @@ -296,6 +296,7 @@ startSession: {skip: "does not return user data"}, top: {skip: "does not return user data"}, touch: {skip: "does not return user data"}, + truncate: {skip: "primary only"}, unsetSharding: {skip: "does not return user data"}, update: {skip: "primary only"}, updateRole: {skip: "primary only"}, diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp index 309393dd40..d1886703f0 100644 --- a/src/mongo/db/catalog/collection_impl.cpp +++ b/src/mongo/db/catalog/collection_impl.cpp @@ -761,43 +761,22 @@ uint64_t CollectionImpl::getIndexSize(OperationContext* opCtx, BSONObjBuilder* d /** * order will be: - * 1) store index specs - * 2) drop indexes - * 3) truncate record store - * 4) re-write indexes + * 1) truncate indices + * 2) truncate record store */ Status CollectionImpl::truncate(OperationContext* opCtx) { dassert(opCtx->lockState()->isCollectionLockedForMode(ns().toString(), MODE_X)); BackgroundOperation::assertNoBgOpInProgForNs(ns()); invariant(_indexCatalog.numIndexesInProgress(opCtx) == 0); - // 1) store index specs - vector indexSpecs; - { - IndexCatalog::IndexIterator ii = _indexCatalog.getIndexIterator(opCtx, false); - while (ii.more()) { - const IndexDescriptor* idx = ii.next(); - indexSpecs.push_back(idx->infoObj().getOwned()); - } + IndexCatalog::IndexIterator ii = _indexCatalog.getIndexIterator(opCtx, false); + while (ii.more()) { + const IndexDescriptor* idx = ii.next(); + ii.accessMethod(idx)->truncate(opCtx); } - - // 2) drop indexes - _indexCatalog.dropAllIndexes(opCtx, true); _cursorManager.invalidateAll(opCtx, false, "collection truncated"); - // 3) truncate record store - auto status = _recordStore->truncate(opCtx); - if (!status.isOK()) - return status; - - // 4) re-create indexes - for (size_t i = 0; i < indexSpecs.size(); i++) { - status = _indexCatalog.createIndexOnEmptyCollection(opCtx, indexSpecs[i]).getStatus(); - if (!status.isOK()) - return status; - } - - return Status::OK(); + return _recordStore->truncate(opCtx); } void CollectionImpl::cappedTruncateAfter(OperationContext* opCtx, RecordId end, bool inclusive) { diff --git a/src/mongo/db/catalog/uuid_catalog.h b/src/mongo/db/catalog/uuid_catalog.h index b561e1d1a3..5315268d59 100644 --- a/src/mongo/db/catalog/uuid_catalog.h +++ b/src/mongo/db/catalog/uuid_catalog.h @@ -119,6 +119,9 @@ public: void onTransactionAbort(OperationContext* opCtx) override {} void onReplicationRollback(OperationContext* opCtx, const RollbackObserverInfo& rbInfo) override {} + void onTruncate(OperationContext* opCtx, + const NamespaceString& nss, + OptionalCollectionUUID uuid) override {} }; /** diff --git a/src/mongo/db/commands/SConscript b/src/mongo/db/commands/SConscript index 1cd906b238..9a6da8fc87 100644 --- a/src/mongo/db/commands/SConscript +++ b/src/mongo/db/commands/SConscript @@ -1,4 +1,4 @@ -# -*- mode: python -*- +#- * - mode : python - * - Import("env") Import("has_option") @@ -87,7 +87,7 @@ env.Library( ], ) -# Commands available in every process that executes commands +#Commands available in every process that executes commands env.Library( target="core", source=[ @@ -200,7 +200,20 @@ env.Library( ], ) -# Commands that are present in both mongod and embedded +env.Library( + target='command_helpers', + source=[ + 'truncate.cpp' + ], + LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/storage/storage_engine_common', + '$BUILD_DIR/mongo/db/catalog/catalog_helpers', + '$BUILD_DIR/mongo/db/commands', + ], +) + +#Commands that are present in both mongod and embedded env.Library( target="standalone", source=[ @@ -251,10 +264,11 @@ env.Library( 'kill_common', 'list_collections_filter', 'write_commands_common', + 'command_helpers', ], ) -# Commands required by the shell to connect +#Commands required by the shell to connect env.Library( target="shell_protocol", source=[ @@ -270,7 +284,7 @@ env.Library( ) -# Commands that should only be present in mongod +#Commands that should only be present in mongod env.Library( target="mongod", source=[ diff --git a/src/mongo/db/commands/truncate.cpp b/src/mongo/db/commands/truncate.cpp new file mode 100644 index 0000000000..65e09ec10f --- /dev/null +++ b/src/mongo/db/commands/truncate.cpp @@ -0,0 +1,138 @@ +/** + * Copyright (C) 2018 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kCommand + +#include "mongo/db/commands/truncate.h" +#include "mongo/base/status.h" +#include "mongo/db/auth/authorization_manager_global.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/catalog/uuid_catalog.h" +#include "mongo/db/command_generic_argument.h" +#include "mongo/db/commands.h" +#include "mongo/db/concurrency/write_conflict_exception.h" +#include "mongo/db/db_raii.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/op_observer.h" +#include "mongo/db/operation_context.h" +#include "mongo/db/repl/oplog.h" +#include "mongo/db/service_context.h" +#include "mongo/rpc/reply_builder_interface.h" +#include "mongo/util/log.h" + +namespace mongo { +Status truncateCommand_impl(OperationContext* opCtx, NamespaceString& nss, bool doStats) { + AutoGetCollection ctx(opCtx, nss, MODE_X, AutoGetCollection::ViewMode::kViewsForbidden); + Collection* coll = ctx.getCollection(); + boost::optional statsTracker; + const boost::optional dbProfilingLevel = boost::none; + if (doStats) { + statsTracker.emplace(opCtx, nss, Top::LockType::WriteLocked, dbProfilingLevel); + log() << "Emplaced statsTracker"; + } + + if (!coll) { // Non-existant collections are fine per dropDatabase and Asya's comment + return Status::OK(); + } + OptionalCollectionUUID uuid = coll->uuid(); + // TODO: return early if empty + return writeConflictRetry(opCtx, "truncate", nss.ns(), [opCtx, coll, nss, uuid] { + WriteUnitOfWork wuow(opCtx); + Status truncateRes = coll->truncate(opCtx); + getGlobalServiceContext()->getOpObserver()->onTruncate(opCtx, nss, uuid); + wuow.commit(); + return truncateRes; + }); + // TODO: Write Conflict Retry +} + +class TruncateInvocation : public CommandInvocation { + NamespaceString _ns; + +public: + TruncateInvocation(Command* def, NamespaceString ns) : CommandInvocation(def), _ns(ns) {} + void run(OperationContext* opCtx, rpc::ReplyBuilderInterface* result) override { + uassert(50916, + "Server not in the correct FCV, need >= 4.2", + serverGlobalParams.featureCompatibility.getVersion() >= + ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo42); + uassertStatusOK(truncateCommand_impl(opCtx, _ns, true)); + } + NamespaceString ns() const override { + return _ns; + } + + bool supportsWriteConcern() const override { + return true; + } + +private: + void doCheckAuthorization(OperationContext* opCtx) const override { + AuthorizationSession* authzSession = AuthorizationSession::get(opCtx->getClient()); + uassert(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to remove documents from collection: " << _ns.ns(), + authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forExactNamespace(_ns), ActionType::remove)); + } +}; +class TruncateCmd : public Command { +public: + TruncateCmd() : Command("truncate") {} + std::unique_ptr parse(OperationContext* opCtx, + const OpMsgRequest& request) override { + const auto nss = AutoGetCollection::resolveNamespaceStringOrUUID( + opCtx, CommandHelpers::parseNsCollectionRequired(request.getDatabase(), request.body)); + for (const BSONElement& e : request.body) { + uassert(50915, + "Invalid argument " + e.fieldNameStringData(), + isGenericArgument(e.fieldNameStringData()) || + e.fieldNameStringData() == "truncate"_sd); + } + uassert(50914, + "Cannot truncate server configuration tables", + nss != NamespaceString::kServerConfigurationNamespace); + return std::make_unique(this, nss); + }; + AllowedOnSecondary secondaryAllowed(ServiceContext* context) const override { + return AllowedOnSecondary::kNever; + } + std::string help() const override { + return "Drops all records in a collection without affecting metadata like indices, etc."; + } + + ReadWriteType getReadWriteType() const override { + return ReadWriteType::kWrite; + } +}; + +MONGO_INITIALIZER_WITH_PREREQUISITES(SetupTruncateCommand, MONGO_NO_PREREQUISITES) +(InitializerContext* context) { + new TruncateCmd(); + return Status::OK(); +} +} // namespace mongo diff --git a/src/mongo/db/commands/truncate.h b/src/mongo/db/commands/truncate.h new file mode 100644 index 0000000000..d825e6745a --- /dev/null +++ b/src/mongo/db/commands/truncate.h @@ -0,0 +1,40 @@ +/** +* Copyright 2018 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General PublicLicense +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once +#include "mongo/base/status.h" +#include "mongo/db/commands.h" +#include "mongo/db/operation_context.h" + +namespace mongo { + +/** + * Implementation of the truncate command used by the oplog and the command invocation. + */ +Status truncateCommand_impl(OperationContext* opCtx, NamespaceString& nss, bool doStats = false); +} diff --git a/src/mongo/db/free_mon/free_mon_op_observer.h b/src/mongo/db/free_mon/free_mon_op_observer.h index 7b4d5861a2..2fca5f7f48 100644 --- a/src/mongo/db/free_mon/free_mon_op_observer.h +++ b/src/mongo/db/free_mon/free_mon_op_observer.h @@ -138,6 +138,10 @@ public: void onTransactionAbort(OperationContext* opCtx) final {} void onReplicationRollback(OperationContext* opCtx, const RollbackObserverInfo& rbInfo); + + void onTruncate(OperationContext* opCtx, + const NamespaceString& nss, + OptionalCollectionUUID uuid) override {} }; } // namespace mongo diff --git a/src/mongo/db/index/index_access_method.cpp b/src/mongo/db/index/index_access_method.cpp index 0dbe51ffbc..86c3d590da 100644 --- a/src/mongo/db/index/index_access_method.cpp +++ b/src/mongo/db/index/index_access_method.cpp @@ -234,6 +234,10 @@ Status IndexAccessMethod::remove(OperationContext* opCtx, return Status::OK(); } +void IndexAccessMethod::truncate(OperationContext* opCtx) { + _newInterface->truncate(opCtx); +} + Status IndexAccessMethod::initializeAsEmpty(OperationContext* opCtx) { return _newInterface->initAsEmpty(opCtx); } diff --git a/src/mongo/db/index/index_access_method.h b/src/mongo/db/index/index_access_method.h index 07d8fa28c7..57c1daee8e 100644 --- a/src/mongo/db/index/index_access_method.h +++ b/src/mongo/db/index/index_access_method.h @@ -94,6 +94,12 @@ public: const InsertDeleteOptions& options, int64_t* numDeleted); + + /** + * Removes all entries from the underlying index. + */ + void truncate(OperationContext* opCtx); + /** * Checks whether the index entries for the document 'from', which is placed at location * 'loc' on disk, can be changed to the index entries for the doc 'to'. Provides a ticket diff --git a/src/mongo/db/op_observer.h b/src/mongo/db/op_observer.h index ebc0590bad..520d4364e6 100644 --- a/src/mongo/db/op_observer.h +++ b/src/mongo/db/op_observer.h @@ -328,6 +328,13 @@ public: virtual void onReplicationRollback(OperationContext* opCtx, const RollbackObserverInfo& rbInfo) = 0; + + /** + * This will get called within the WUOW of the truncate command. + */ + virtual void onTruncate(OperationContext* opCtx, + const NamespaceString& nss, + OptionalCollectionUUID uuid) = 0; struct Times; protected: diff --git a/src/mongo/db/op_observer_impl.cpp b/src/mongo/db/op_observer_impl.cpp index 9117ce32e6..33bcf85f20 100644 --- a/src/mongo/db/op_observer_impl.cpp +++ b/src/mongo/db/op_observer_impl.cpp @@ -58,6 +58,7 @@ namespace mongo { using repl::OplogEntry; +using repl::Truncate; namespace { MONGO_FAIL_POINT_DEFINE(failCollectionUpdates); @@ -928,6 +929,27 @@ void OpObserverImpl::onEmptyCapped(OperationContext* opCtx, ->logOp(opCtx, "c", cmdNss, cmdObj, nullptr); } +void OpObserverImpl::onTruncate(OperationContext* opCtx, + const NamespaceString& nss, + OptionalCollectionUUID uuid) { + const auto cmdObj = BSON(Truncate::kNamespaceFieldName << nss.coll()); + logOperation(opCtx, + "c", + nss.getCommandNS(), + uuid, + cmdObj, + nullptr, + false, + getWallClockTimeForOpLog(opCtx), + {}, + kUninitializedStmtId, + {}, + false /* prepare */, + OplogSlot()); + + AuthorizationManager::get(opCtx->getServiceContext())->logOp(opCtx, "c", nss, cmdObj, nullptr); +} + namespace { OpTimeBundle logApplyOpsForTransaction(OperationContext* opCtx, diff --git a/src/mongo/db/op_observer_impl.h b/src/mongo/db/op_observer_impl.h index 1fa7e076eb..dc3f2dbd70 100644 --- a/src/mongo/db/op_observer_impl.h +++ b/src/mongo/db/op_observer_impl.h @@ -115,6 +115,9 @@ public: void onTransactionAbort(OperationContext* opCtx) override; void onReplicationRollback(OperationContext* opCtx, const RollbackObserverInfo& rbInfo) override; + void onTruncate(OperationContext* opCtx, + const NamespaceString& nss, + OptionalCollectionUUID uuid); }; } // namespace mongo diff --git a/src/mongo/db/op_observer_noop.h b/src/mongo/db/op_observer_noop.h index 98be250f0c..7b34977a4e 100644 --- a/src/mongo/db/op_observer_noop.h +++ b/src/mongo/db/op_observer_noop.h @@ -114,6 +114,9 @@ public: void onTransactionAbort(OperationContext* opCtx) override{}; void onReplicationRollback(OperationContext* opCtx, const RollbackObserverInfo& rbInfo) override {} + void onTruncate(OperationContext* opCtx, + const NamespaceString& nss, + OptionalCollectionUUID uuid) override {} }; } // namespace mongo diff --git a/src/mongo/db/op_observer_registry.h b/src/mongo/db/op_observer_registry.h index a89950d81c..831329f5a6 100644 --- a/src/mongo/db/op_observer_registry.h +++ b/src/mongo/db/op_observer_registry.h @@ -240,6 +240,13 @@ public: for (auto& o : _observers) o->onReplicationRollback(opCtx, rbInfo); } + void onTruncate(OperationContext* opCtx, + const NamespaceString& nss, + OptionalCollectionUUID uuid) override { + ReservedTimes times{opCtx}; + for (auto& o : _observers) + o->onTruncate(opCtx, nss, uuid); + } private: static repl::OpTime _getOpTimeToReturn(const std::vector& times) { diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript index a5f6656495..adb99f9bcb 100644 --- a/src/mongo/db/repl/SConscript +++ b/src/mongo/db/repl/SConscript @@ -18,6 +18,7 @@ env.Library( 'repl_settings', '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/background', + '$BUILD_DIR/mongo/db/commands/command_helpers', '$BUILD_DIR/mongo/db/catalog/catalog_helpers', '$BUILD_DIR/mongo/db/commands/feature_compatibility_parsers', '$BUILD_DIR/mongo/db/db_raii', diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp index 93906fc21a..8d7aebd075 100644 --- a/src/mongo/db/repl/oplog.cpp +++ b/src/mongo/db/repl/oplog.cpp @@ -57,6 +57,7 @@ #include "mongo/db/client.h" #include "mongo/db/commands.h" #include "mongo/db/commands/feature_compatibility_version_parser.h" +#include "mongo/db/commands/truncate.h" #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db_raii.h" #include "mongo/db/dbdirectclient.h" @@ -963,6 +964,16 @@ std::map opsMap = { return emptyCapped(opCtx, parseUUIDorNs(opCtx, ns, ui, cmd)); }, {ErrorCodes::NamespaceNotFound}}}, + {"truncate", + {[](OperationContext* opCtx, + const char* ns, + const BSONElement& ui, + BSONObj& cmdObj, + const OpTime& opTime, + OplogApplication::Mode mode) -> Status { + auto nss = parseUUIDorNs(opCtx, ns, ui, cmdObj); + return truncateCommand_impl(opCtx, nss); + }}}, }; } // namespace diff --git a/src/mongo/db/repl/oplog_entry.cpp b/src/mongo/db/repl/oplog_entry.cpp index 32e2880c12..2705506455 100644 --- a/src/mongo/db/repl/oplog_entry.cpp +++ b/src/mongo/db/repl/oplog_entry.cpp @@ -68,6 +68,8 @@ OplogEntry::CommandType parseCommandType(const BSONObj& objectField) { return OplogEntry::CommandType::kDropIndexes; } else if (commandString == "commitTransaction") { return OplogEntry::CommandType::kCommitTransaction; + } else if (commandString == "truncate") { + return OplogEntry::CommandType::kTruncate; } else { severe() << "Unknown oplog entry command type: " << commandString << " Object field: " << redact(objectField); diff --git a/src/mongo/db/repl/oplog_entry.h b/src/mongo/db/repl/oplog_entry.h index 04e6a56781..07f69a3218 100644 --- a/src/mongo/db/repl/oplog_entry.h +++ b/src/mongo/db/repl/oplog_entry.h @@ -57,6 +57,7 @@ public: kCreateIndexes, kDropIndexes, kCommitTransaction, + kTruncate }; // Current oplog version, should be the value of the v field in all oplog entries. diff --git a/src/mongo/db/repl/oplog_entry.idl b/src/mongo/db/repl/oplog_entry.idl index 9d0af006a9..6e6cc9c55e 100644 --- a/src/mongo/db/repl/oplog_entry.idl +++ b/src/mongo/db/repl/oplog_entry.idl @@ -143,3 +143,10 @@ structs: type: bool optional: true description: "Specifies that this operation should be put into a 'prepare' state" + Truncate: + description: Truncate command. + fields: + truncate: + cpp_name: namespace + type: namespacestring + description: "The collection to truncate" diff --git a/src/mongo/db/s/config_server_op_observer.h b/src/mongo/db/s/config_server_op_observer.h index ebed73cce5..3a71d0526a 100644 --- a/src/mongo/db/s/config_server_op_observer.h +++ b/src/mongo/db/s/config_server_op_observer.h @@ -138,6 +138,10 @@ public: void onTransactionAbort(OperationContext* opCtx) override {} void onReplicationRollback(OperationContext* opCtx, const RollbackObserverInfo& rbInfo); + + void onTruncate(OperationContext* opCtx, + const NamespaceString& nss, + OptionalCollectionUUID uuid) override {} }; } // namespace mongo diff --git a/src/mongo/db/s/shard_server_op_observer.h b/src/mongo/db/s/shard_server_op_observer.h index 9edc8da25f..f3bf8c4cee 100644 --- a/src/mongo/db/s/shard_server_op_observer.h +++ b/src/mongo/db/s/shard_server_op_observer.h @@ -139,6 +139,10 @@ public: void onTransactionAbort(OperationContext* opCtx) override {} void onReplicationRollback(OperationContext* opCtx, const RollbackObserverInfo& rbInfo) {} + + void onTruncate(OperationContext* opCtx, + const NamespaceString& nss, + OptionalCollectionUUID uuid) override {} }; diff --git a/src/mongo/db/storage/biggie/biggie_sorted_impl.cpp b/src/mongo/db/storage/biggie/biggie_sorted_impl.cpp index 66beebd5ab..b526747593 100644 --- a/src/mongo/db/storage/biggie/biggie_sorted_impl.cpp +++ b/src/mongo/db/storage/biggie/biggie_sorted_impl.cpp @@ -374,7 +374,7 @@ void SortedDataInterface::unindex(OperationContext* opCtx, // This function is, as of now, not in the interface, but there exists a server ticket to add // truncate to the list of commands able to be used. -Status SortedDataInterface::truncate(OperationContext* opCtx) { +void SortedDataInterface::truncate(OperationContext* opCtx) { StringStore* workingCopy = getRecoveryUnitBranch_forking(opCtx); auto workingCopyLowerBound = workingCopy->lower_bound(_KSForIdentStart); auto workingCopyUpperBound = workingCopy->upper_bound(_KSForIdentEnd); @@ -383,7 +383,6 @@ Status SortedDataInterface::truncate(OperationContext* opCtx) { workingCopy->erase(workingCopyLowerBound->first); ++workingCopyLowerBound; } - return Status::OK(); } Status SortedDataInterface::dupKeyCheck(OperationContext* opCtx, diff --git a/src/mongo/db/storage/biggie/biggie_sorted_impl.h b/src/mongo/db/storage/biggie/biggie_sorted_impl.h index 468cb27ab9..d928c18d36 100644 --- a/src/mongo/db/storage/biggie/biggie_sorted_impl.h +++ b/src/mongo/db/storage/biggie/biggie_sorted_impl.h @@ -66,7 +66,7 @@ class SortedDataInterface : public ::mongo::SortedDataInterface { public: // Truncate is not required at the time of writing but will be when the truncate command is // created - Status truncate(OperationContext* opCtx); + void truncate(OperationContext* opCtx) override; SortedDataInterface(const Ordering& ordering, bool isUnique, StringData ident); virtual SortedDataBuilderInterface* getBulkBuilder(OperationContext* opCtx, bool dupsAllowed) override; diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp index 94a8c1bc4d..ff60443675 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp @@ -216,6 +216,8 @@ public: const RecordId& loc, bool dupsAllowed) {} + void truncate(OperationContext* opCtx) override {} + virtual Status dupKeyCheck(OperationContext* opCtx, const BSONObj& key, const RecordId& loc) { return Status::OK(); } diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.cpp b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.cpp index 75c30bc0f6..9b898cb443 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.cpp +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.cpp @@ -188,6 +188,10 @@ public: } } + void truncate(OperationContext* opCtx) override { + opCtx->recoveryUnit()->registerChange(new TruncateChange(opCtx, _data)); + } + virtual void fullValidate(OperationContext* opCtx, long long* numKeysOut, ValidateResults* fullResults) const { @@ -499,6 +503,24 @@ private: const IndexKeyEntry _entry; const bool _insert; }; + class TruncateChange : public RecoveryUnit::Change { + public: + TruncateChange(OperationContext* opCtx, IndexSet* data) + : _opCtx(opCtx), _data(data), _localCopy(new IndexSet(_data->key_comp())) { + std::swap(_data, _localCopy); + } + virtual void commit(boost::optional) { + delete _localCopy; + } + virtual void rollback() { + std::swap(_data, _localCopy); // swap back + } + + private: + OperationContext* _opCtx; + IndexSet* _data; + IndexSet* _localCopy; + }; IndexSet* _data; long long _currentKeySize; diff --git a/src/mongo/db/storage/mobile/mobile_index.cpp b/src/mongo/db/storage/mobile/mobile_index.cpp index bf61b8fe5c..b7e1639791 100644 --- a/src/mongo/db/storage/mobile/mobile_index.cpp +++ b/src/mongo/db/storage/mobile/mobile_index.cpp @@ -139,6 +139,12 @@ void MobileIndex::unindex(OperationContext* opCtx, return _unindex(opCtx, key, recId, dupsAllowed); } +void MobileIndex::truncate(OperationContext* opCtx) { + MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false); + std::string deleteForTruncateQuery = "DELETE FROM \"" + _ident + "\";"; + SqliteStatement::execQuery(session, deleteForTruncateQuery); +} + void MobileIndex::_doDelete(OperationContext* opCtx, const KeyString& key, KeyString* value) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false); diff --git a/src/mongo/db/storage/mobile/mobile_index.h b/src/mongo/db/storage/mobile/mobile_index.h index a22ba4f4e1..9131cccd0d 100644 --- a/src/mongo/db/storage/mobile/mobile_index.h +++ b/src/mongo/db/storage/mobile/mobile_index.h @@ -57,6 +57,8 @@ public: const RecordId& recId, bool dupsAllowed) override; + void truncate(OperationContext* opCtx) override; + void fullValidate(OperationContext* opCtx, long long* numKeysOut, ValidateResults* fullResults) const override; diff --git a/src/mongo/db/storage/sorted_data_interface.h b/src/mongo/db/storage/sorted_data_interface.h index aa6d1e6e51..af3a0d0e8b 100644 --- a/src/mongo/db/storage/sorted_data_interface.h +++ b/src/mongo/db/storage/sorted_data_interface.h @@ -113,6 +113,11 @@ public: const RecordId& loc, bool dupsAllowed) = 0; + /** + * Removes all entries from the index. + */ + virtual void truncate(OperationContext* opCtx) = 0; + /** * Return ErrorCodes::DuplicateKey if 'key' already exists in 'this' * index at a RecordId other than 'loc', and Status::OK() otherwise. diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp index 72d7c85126..c6890ce1ee 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp @@ -348,6 +348,18 @@ void WiredTigerIndex::unindex(OperationContext* opCtx, _unindex(opCtx, c, key, id, dupsAllowed); } +void WiredTigerIndex::truncate(OperationContext* opCtx) { + WiredTigerCursor curwrap(_uri, _tableId, false, opCtx); + curwrap.assertInActiveTxn(); + WT_CURSOR* c = curwrap.get(); + int ret = wiredTigerPrepareConflictRetry(opCtx, [&] { return c->next(c); }); + if (ret == WT_NOTFOUND) { + return; + } + WT_SESSION* session = WiredTigerRecoveryUnit::get(opCtx)->getSession()->getSession(); + invariantWTOK(WT_OP_CHECK(session->truncate(session, NULL, c, NULL, NULL))); +} + void WiredTigerIndex::fullValidate(OperationContext* opCtx, long long* numKeysOut, ValidateResults* fullResults) const { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h index 5e471d5923..81b23e1e07 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h @@ -100,6 +100,8 @@ public: const RecordId& id, bool dupsAllowed); + virtual void truncate(OperationContext* opCtx) override; + virtual void fullValidate(OperationContext* opCtx, long long* numKeysOut, ValidateResults* fullResults) const; diff --git a/src/mongo/embedded/capi_test.cpp b/src/mongo/embedded/capi_test.cpp index 6de0f304f1..a663645ab5 100644 --- a/src/mongo/embedded/capi_test.cpp +++ b/src/mongo/embedded/capi_test.cpp @@ -598,6 +598,7 @@ TEST_F(MongodbCAPITest, RunListCommands) { "setParameter", "sleep", "trimMemory", + "truncate", "update", "validate"}; std::sort(whitelist.begin(), whitelist.end()); diff --git a/src/mongo/s/commands/SConscript b/src/mongo/s/commands/SConscript index 2c642d077b..51d6a9c503 100644 --- a/src/mongo/s/commands/SConscript +++ b/src/mongo/s/commands/SConscript @@ -92,6 +92,7 @@ env.Library( 'cluster_shard_collection_cmd.cpp', 'cluster_shutdown_cmd.cpp', 'cluster_split_cmd.cpp', + 'cluster_truncate_cmd.cpp', 'cluster_update_zone_key_range_cmd.cpp', 'cluster_user_management_commands.cpp', 'cluster_validate_cmd.cpp', diff --git a/src/mongo/s/commands/cluster_truncate_cmd.cpp b/src/mongo/s/commands/cluster_truncate_cmd.cpp new file mode 100644 index 0000000000..a388650122 --- /dev/null +++ b/src/mongo/s/commands/cluster_truncate_cmd.cpp @@ -0,0 +1,125 @@ +/** + * Copyright (C) 2018 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include + +#include "mongo/db/auth/authorization_manager_global.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/command_generic_argument.h" +#include "mongo/db/commands.h" +#include "mongo/db/db_raii.h" +#include "mongo/rpc/get_status_from_command_result.h" +#include "mongo/s/commands/cluster_commands_helpers.h" +#include "mongo/s/grid.h" + +namespace mongo { +class ClusterTruncateInvocation : public CommandInvocation { + BSONObj _cmdObj; + NamespaceString _ns; + +public: + ClusterTruncateInvocation(Command* def, BSONObj cmdObj, NamespaceString ns) + : CommandInvocation(def), _cmdObj(cmdObj), _ns(ns) {} + void run(OperationContext* opCtx, rpc::ReplyBuilderInterface* result) override { + // Leave FCV checks to mongod + auto routingInfo = + uassertStatusOK(Grid::get(opCtx)->catalogCache()->getCollectionRoutingInfo(opCtx, _ns)); + auto shardResults = scatterGatherVersionedTargetByRoutingTable( + opCtx, + _ns.db(), + _ns, + routingInfo, + CommandHelpers::filterCommandRequestForPassthrough(_cmdObj), + ReadPreferenceSetting::get(opCtx), + Shard::RetryPolicy::kIdempotent, + {}, + {}); + for (const auto& shardRes : shardResults) { + const auto shardResponse = uassertStatusOK(std::move(shardRes.swResponse)); + uassertStatusOK(shardResponse.status); + const auto& res = shardResponse.data; + uassertStatusOK(getStatusFromCommandResult(res)); + } + // TODO: should this conform to db.dropDatabase() or db.foo.drop() ? + result->setCommandReply(StatusWith(BSON("truncated" << _ns.ns()))); + } + NamespaceString ns() const override { + return _ns; + } + + bool supportsWriteConcern() const override { + return true; + } + +private: + void doCheckAuthorization(OperationContext* opCtx) const override { + AuthorizationSession* authzSession = AuthorizationSession::get(opCtx->getClient()); + uassert(ErrorCodes::Unauthorized, + str::stream() << "Not authorized to remove documents from collection: " << _ns.ns(), + authzSession->isAuthorizedForActionsOnResource( + ResourcePattern::forExactNamespace(_ns), ActionType::remove)); + } +}; +class ClusterTruncateCmd : public Command { +public: + ClusterTruncateCmd() : Command("truncate") {} + std::unique_ptr parse(OperationContext* opCtx, + const OpMsgRequest& request) override { + StringData dbName(request.getDatabase()); + NamespaceString nss(CommandHelpers::parseNsFromCommand(dbName, request.body)); + for (const BSONElement& e : request.body) { + uassert(50911, + "Invalid argument " + e.fieldNameStringData(), + isGenericArgument(e.fieldNameStringData()) || + e.fieldNameStringData() == "truncate"_sd); + } + uassert(50910, + "Cannot truncate server configuration tables", + nss != NamespaceString::kServerConfigurationNamespace); + return std::make_unique(this, request.body, nss); + }; + AllowedOnSecondary secondaryAllowed(ServiceContext* context) const override { + return AllowedOnSecondary::kNever; + } + std::string help() const override { + return "Drops all records in a collection without affecting metadata like indices, etc."; + } + + ReadWriteType getReadWriteType() const override { + return ReadWriteType::kWrite; + } +}; + +MONGO_INITIALIZER_WITH_PREREQUISITES(SetupClusterTruncateCommand, MONGO_NO_PREREQUISITES) +(InitializerContext* context) { + new ClusterTruncateCmd(); + return Status::OK(); +} +} -- 2.14.4