-
Type:
Bug
-
Resolution: Unresolved
-
Priority:
Major - P3
-
None
-
Affects Version/s: None
-
Component/s: None
-
None
-
Server Security
-
ALL
-
None
-
None
-
None
-
None
-
None
-
None
-
None
The cleanup and compact operations on queryable encryption collections are internally renaming and then recreating the ECOC collection.
Since the ECOC collection lock is released at the end of the rename and reacquired at the beginning of the create, in presence of concurrent user insert/update it is likely for the collection to be implicitly created with default options (hence missing the clustered index and resulting in performance degradation). This is highly probable because user writes would serialize on the collection lock behind the rename and happen right when it is released, sneaking in before the create.
When that happens, the internal create performed by cleanup and compact silently fails and the original operation returns success to the user.
Reproducible (apply patch on top of 5337826dd8e3fdf50473cce3804130e1612abac2):
diff --git a/src/mongo/db/commands/fle2_compact_cmd.cpp b/src/mongo/db/commands/fle2_compact_cmd.cpp
index 4c90ec23e27..d4d6484105e 100644
--- a/src/mongo/db/commands/fle2_compact_cmd.cpp
+++ b/src/mongo/db/commands/fle2_compact_cmd.cpp
@@ -188,7 +188,7 @@ CompactStats compactEncryptedCompactionCollection(OperationContext* opCtx,
"Resuming compaction from a stale ECOC collection",
logAttrs(namespaces.ecocRenameNss));
}
-
+ sleepFor(Milliseconds(5000));
if (!ecoc) {
if (MONGO_unlikely(fleCompactHangBeforeECOCCreateUnsharded.shouldFail())) {
LOGV2(7299601, "Hanging due to fleCompactHangBeforeECOCCreateUnsharded fail point");
diff --git a/jstests/fle2/insert_encrypted_documents.js b/jstests/fle2/insert_encrypted_documents.js
new file mode 100644
index 00000000000..d51e08a9ec0
--- /dev/null
+++ b/jstests/fle2/insert_encrypted_documents.js
@@ -0,0 +1,88 @@
+/**
+ * Creates a QE collection, starts compaction and concurrently inserts 10 documents with encrypted fields.
+ *
+ * @tags: [
+ * no_selinux,
+ * does_not_support_transactions,
+ * does_not_support_stepdowns,
+ * ]
+ */
+import {
+ EncryptedClient,
+ isEnterpriseShell,
+} from "jstests/fle2/libs/encrypted_client_util.js";
+import {funWithArgs} from "jstests/libs/parallel_shell_helpers.js";
+
+if (!isEnterpriseShell()) {
+ jsTestLog("Skipping test as it requires the enterprise module");
+ quit();
+}
+
+const dbName = "test";
+const collName = "coll";
+
+function getEcocOptions(){
+ return db.runCommand({listCollections: 1, filter: {name: 'enxcol_.' + collName + '.ecoc'}}).cursor.firstBatch[0].options;
+}
+
+const testDB = db.getSiblingDB(dbName);
+testDB.dropDatabase();
+
+const client = new EncryptedClient(db.getMongo(), dbName);
+
+assert.commandWorked(
+ client.createEncryptionCollection(collName, {
+ encryptedFields: {
+ fields: [
+ {path: "ssn", bsonType: "string", queries: {queryType: "equality"}}
+ ],
+ },
+ }),
+);
+
+assert.neq(undefined, getEcocOptions().clusteredIndex);
+
+// Run compactStructuredEncryptionData in a parallel shell. The shell bootstraps its own
+// encrypted connection (same local KMS key + key vault) so it can generate the compaction
+// tokens the command requires.
+const runCompact = function(dbName, collName) {
+ const conn = db.getMongo();
+ conn.setAutoEncryption({
+ kmsProviders: {
+ local: {
+ key: BinData(
+ 0,
+ "/tu9jUCBqZdwCelwE/EAm/4WqdxrSMi04B8e9uAV+m30rI1J2nhKZZtQjdvsSCwuI4erR6IEcEK+5eGUAODv43NDNIR9QheT2edWFewUfHKsl9cnzTc86meIzOmYl6dr",
+ ),
+ },
+ },
+ keyVaultNamespace: dbName + ".keystore",
+ schemaMap: {},
+ });
+ conn.toggleAutoEncryption(true);
+ const reply = assert.commandWorked(db.getSiblingDB(dbName)[collName].compact());
+ jsTestLog("compactStructuredEncryptionData reply: " + tojson(reply));
+ conn.toggleAutoEncryption(false);
+ conn.unsetAutoEncryption();
+};
+const joinCompact = startParallelShell(funWithArgs(runCompact, dbName, collName), db.getMongo().port);
+
+client.runEncryptionOperation(() => {
+ const ecoll = client.getDB()[collName];
+
+ for (let i = 0; i < 10; i++) {
+ const ssn = `${String(i).padStart(3, "0")}-00-0000`;
+ assert.commandWorked(ecoll.insertOne({
+ _id: i,
+ ssn: ssn,
+ }));
+ sleep(1000);
+ }
+});
+
+jsTestLog("Waiting for compactStructuredEncryptionData to finish");
+joinCompact();
+
+// This will fail because the ECOC collection has been implicitly created
+// by an insert while compaction was ongoing
+assert.neq(undefined, getEcocOptions().clusteredIndex, "ECOC collection missing clustered index");
To execute the reproducible:
./buildscripts/resmoke.py run --suite=fle2 --mongodSetParameters='{logComponentVerbosity: {verbosity: 1}}' --storageEngine=wiredTiger --jobs=1 --storageEngineCacheSizeGB=0.5 --dbpath=/tmp/testpath jstests/fle2/insert_encrypted_documents.js >| out.txt