[SERVER-41768] Inclusion projection scenario that falsely writes included field to document Created: 14/Jun/19  Updated: 24/Feb/20  Resolved: 21/Jun/19

Status: Closed
Project: Core Server
Component/s: Aggregation Framework
Affects Version/s: 4.0.10, 4.3.1
Fix Version/s: None

Type: Bug Priority: Major - P3
Reporter: James Wahlin Assignee: James Wahlin
Resolution: Duplicate Votes: 0
Labels: afz
Remaining Estimate: Not Specified
Time Spent: Not Specified
Original Estimate: Not Specified

Issue Links:
Duplicate
duplicates SERVER-23229 Projection incorrectly returns null v... Backlog
Operating System: ALL
Steps To Reproduce:

db.coll.drop();
db.coll.insert({});
db.coll.createIndex({"str": 1});
 
var result =
    db.coll.aggregate([{$match: {'str': {$not: {$eq: ''}}}},
                                         {$project: {"str": 1, "_id": 0}}]).toArray();
assert.eq({}, result[0]);

Participants:

 Description   

In the reproduction below an aggregate with match and inclusion projection writes the included field to the output document. It should instead produce an empty document. This is consistently reproducible on 4.0.10 and occurs sporadically on master and 4.2.0-rc0.

The assertion fails with the following error:

assert: [{ }] != [{ "str" : null }] are not equal



 Comments   
Comment by James Wahlin [ 21/Jun/19 ]

Closing as a duplicate of SERVER-23229.

Comment by James Wahlin [ 14/Jun/19 ]

The following is the explain output for the 4.0.10 reproduction. We are suspicious that this plan has no fetch stage, which would be required to disambiguate whether the index key represents an actual null value or a missing field.

{
    "stages": [{
        "$cursor": {
            "query": {
                "$and": [
                    {
                      "$or":
                          [{"obj.obj.obj.str": {"$not": {"$in": []}}}, {"obj.date": {"$nin": []}}]
                    },
                    {
                      "obj.obj.obj.obj.str":
                          {"$not": {"$lte": "Integration Money Market Account Direct"}}
                    }
                ]
            },
            "fields": {"obj.obj.obj.obj.str": 1, "_id": 0},
            "queryPlanner": {
                "plannerVersion": 1,
                "namespace": "test.test",
                "indexFilterSet": false,
                "parsedQuery": {
                    "$and": [
                        {
                          "$or": [
                              {"obj.date": {"$not": {"$in": []}}},
                              {"obj.obj.obj.str": {"$not": {"$in": []}}}
                          ]
                        },
                        {
                          "obj.obj.obj.obj.str":
                              {"$not": {"$lte": "Integration Money Market Account Direct"}}
                        }
                    ]
                },
                "winningPlan": {
                    "stage": "PROJECTION",
                    "transformBy": {"obj.obj.obj.obj.str": 1, "_id": 0},
                    "inputStage": {
                        "stage": "OR",
                        "inputStages": [
                            {
                              "stage": "IXSCAN",
                              "keyPattern": {"obj.obj.obj.obj.str": 1, "obj.obj.obj.str": 1},
                              "indexName": "obj.obj.obj.obj.str_1_obj.obj.obj.str_1",
                              "isMultiKey": false,
                              "multiKeyPaths": {"obj.obj.obj.obj.str": [], "obj.obj.obj.str": []},
                              "isUnique": false,
                              "isSparse": false,
                              "isPartial": false,
                              "indexVersion": 2,
                              "direction": "forward",
                              "indexBounds": {
                                  "obj.obj.obj.obj.str": [
                                      "[MinKey, \"\")",
                                      "(\"Integration Money Market Account Direct\", MaxKey]"
                                  ],
                                  "obj.obj.obj.str": ["[MinKey, MaxKey]"]
                              }
                            },
                            {
                              "stage": "IXSCAN",
                              "keyPattern": {"obj.obj.obj.obj.str": 1, "obj.date": 1},
                              "indexName": "obj.obj.obj.obj.str_1_obj.date_1",
                              "isMultiKey": false,
                              "multiKeyPaths": {"obj.obj.obj.obj.str": [], "obj.date": []},
                              "isUnique": false,
                              "isSparse": false,
                              "isPartial": false,
                              "indexVersion": 2,
                              "direction": "forward",
                              "indexBounds": {
                                  "obj.obj.obj.obj.str": [
                                      "[MinKey, \"\")",
                                      "(\"Integration Money Market Account Direct\", MaxKey]"
                                  ],
                                  "obj.date": ["[MinKey, MaxKey]"]
                              }
                            }
                        ]
                    }
                },
                "rejectedPlans": []
            },
            "executionStats": {
                "executionSuccess": true,
                "nReturned": 1,
                "executionTimeMillis": 1,
                "totalKeysExamined": 2,
                "totalDocsExamined": 0,
                "executionStages": {
                    "stage": "PROJECTION",
                    "nReturned": 1,
                    "executionTimeMillisEstimate": 0,
                    "works": 4,
                    "advanced": 1,
                    "needTime": 2,
                    "needYield": 0,
                    "saveState": 1,
                    "restoreState": 1,
                    "isEOF": 1,
                    "invalidates": 0,
                    "transformBy": {"obj.obj.obj.obj.str": 1, "_id": 0},
                    "inputStage": {
                        "stage": "OR",
                        "nReturned": 1,
                        "executionTimeMillisEstimate": 0,
                        "works": 4,
                        "advanced": 1,
                        "needTime": 2,
                        "needYield": 0,
                        "saveState": 1,
                        "restoreState": 1,
                        "isEOF": 1,
                        "invalidates": 0,
                        "dupsTested": 2,
                        "dupsDropped": 1,
                        "recordIdsForgotten": 0,
                        "inputStages": [
                            {
                              "stage": "IXSCAN",
                              "nReturned": 1,
                              "executionTimeMillisEstimate": 0,
                              "works": 2,
                              "advanced": 1,
                              "needTime": 0,
                              "needYield": 0,
                              "saveState": 1,
                              "restoreState": 1,
                              "isEOF": 1,
                              "invalidates": 0,
                              "keyPattern": {"obj.obj.obj.obj.str": 1, "obj.obj.obj.str": 1},
                              "indexName": "obj.obj.obj.obj.str_1_obj.obj.obj.str_1",
                              "isMultiKey": false,
                              "multiKeyPaths": {"obj.obj.obj.obj.str": [], "obj.obj.obj.str": []},
                              "isUnique": false,
                              "isSparse": false,
                              "isPartial": false,
                              "indexVersion": 2,
                              "direction": "forward",
                              "indexBounds": {
                                  "obj.obj.obj.obj.str": [
                                      "[MinKey, \"\")",
                                      "(\"Integration Money Market Account Direct\", MaxKey]"
                                  ],
                                  "obj.obj.obj.str": ["[MinKey, MaxKey]"]
                              },
                              "keysExamined": 1,
                              "seeks": 1,
                              "dupsTested": 0,
                              "dupsDropped": 0,
                              "seenInvalidated": 0
                            },
                            {
                              "stage": "IXSCAN",
                              "nReturned": 1,
                              "executionTimeMillisEstimate": 0,
                              "works": 2,
                              "advanced": 1,
                              "needTime": 0,
                              "needYield": 0,
                              "saveState": 1,
                              "restoreState": 1,
                              "isEOF": 1,
                              "invalidates": 0,
                              "keyPattern": {"obj.obj.obj.obj.str": 1, "obj.date": 1},
                              "indexName": "obj.obj.obj.obj.str_1_obj.date_1",
                              "isMultiKey": false,
                              "multiKeyPaths": {"obj.obj.obj.obj.str": [], "obj.date": []},
                              "isUnique": false,
                              "isSparse": false,
                              "isPartial": false,
                              "indexVersion": 2,
                              "direction": "forward",
                              "indexBounds": {
                                  "obj.obj.obj.obj.str": [
                                      "[MinKey, \"\")",
                                      "(\"Integration Money Market Account Direct\", MaxKey]"
                                  ],
                                  "obj.date": ["[MinKey, MaxKey]"]
                              },
                              "keysExamined": 1,
                              "seeks": 1,
                              "dupsTested": 0,
                              "dupsDropped": 0,
                              "seenInvalidated": 0
                            }
                        ]
                    }
                },
                "allPlansExecution": []
            }
        }
    }],
        "ok": 1
}

Generated at Thu Feb 08 04:58:37 UTC 2024 using Jira 9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66.