Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-71270

Time series optimization pushes $match on timeField before $project stage that removes the field from pipeline

    • Type: Icon: Bug Bug
    • Resolution: Fixed
    • Priority: Icon: Major - P3 Major - P3
    • 6.2.0-rc2, 6.3.0-rc0, 6.0.7
    • Affects Version/s: None
    • Component/s: None
    • Labels:
      None
    • Fully Compatible
    • ALL
    • v6.2, v6.0
    • Hide
      const doc = {
        _id: 0,
        time: new Date('2019-01-18T13:24:15.443Z'),
        tag: {},
      };
      
      db.ts.drop();
      db.coll.drop();
      
      db.createCollection('ts', {timeseries: {timeField: 'time', metaField: 'tag'}});
      db.createCollection('coll');
      
      db.ts.insertOne(doc);
      db.coll.insertOne(doc);
      const pipeline = [
        {$project: {'time': 0}},
        {$match: {'time': {$lte: new Date('2019-02-13T11:36:03.481Z')}}},
      ];
      
      const ts = db.ts.aggregate(pipeline).toArray();
      // [ { "tag" : { }, "_id" : 0 } ]
      const vanilla = db.coll.aggregate(pipeline).toArray();
      // [ ]
      
      Show
      const doc = { _id: 0, time: new Date( '2019-01-18T13:24:15.443Z' ), tag: {}, }; db.ts.drop(); db.coll.drop(); db.createCollection( 'ts' , {timeseries: {timeField: 'time' , metaField: 'tag' }}); db.createCollection( 'coll' ); db.ts.insertOne(doc); db.coll.insertOne(doc); const pipeline = [ {$project: { 'time' : 0}}, {$match: { 'time' : {$lte: new Date( '2019-02-13T11:36:03.481Z' )}}}, ]; const ts = db.ts.aggregate(pipeline).toArray(); // [ { "tag" : { }, "_id" : 0 } ] const vanilla = db.coll.aggregate(pipeline).toArray(); // [ ]
    • QO 2022-11-14, QE 2022-11-28
    • 135

      With the query:

      [
        {$project: {'time': 0}},
        {$match: {'time': {$lte: new Date('2019-02-13T11:36:03.481Z')}}},
      ]
      

      The optimized plan looks like:

      "stages" : [
      	{
      		"$cursor" : {
      			"queryPlanner" : {
      				"namespace" : "test.system.buckets.ts",
      				"indexFilterSet" : false,
      				"parsedQuery" : {
      					"$and" : [
      						{
      							"_id" : {
      								"$lte" : ObjectId("5c640124ffffffffffffffff")
      							}
      						},
      						{
      							"control.max.time" : {
      								"$_internalExprLte" : ISODate("2019-02-13T12:36:03.481Z")
      							}
      						},
      						{
      							"control.min.time" : {
      								"$_internalExprLte" : ISODate("2019-02-13T11:36:03.481Z")
      							}
      						}
      					]
      				},
      				"queryHash" : "A79A3A87",
      				"planCacheKey" : "A79A3A87",
      				"maxIndexedOrSolutionsReached" : false,
      				"maxIndexedAndSolutionsReached" : false,
      				"maxScansToExplodeReached" : false,
      				"winningPlan" : {
      					"stage" : "CLUSTERED_IXSCAN",
      					"filter" : {
      						"$and" : [
      							{
      								"_id" : {
      									"$lte" : ObjectId("5c640124ffffffffffffffff")
      								}
      							},
      							{
      								"control.max.time" : {
      									"$_internalExprLte" : ISODate("2019-02-13T12:36:03.481Z")
      								}
      							},
      							{
      								"control.min.time" : {
      									"$_internalExprLte" : ISODate("2019-02-13T11:36:03.481Z")
      								}
      							}
      						]
      					},
      					"direction" : "forward",
      					"minRecord" : ObjectId("000000000000000000000000"),
      					"maxRecord" : ObjectId("5c640124ffffffffffffffff")
      				},
      				"rejectedPlans" : [ ]
      			}
      		}
      	},
      	{
      		"$_internalUnpackBucket" : {
      			"exclude" : [
      				"time"
      			],
      			"timeField" : "time",
      			"metaField" : "tag",
      			"bucketMaxSpanSeconds" : 3600,
      			"assumeNoMixedSchemaData" : true,
      			"wholeBucketFilter" : {
      				"control.max.time" : {
      					"$lte" : ISODate("2019-02-13T11:36:03.481Z")
      				}
      			},
      			"eventFilter" : {
      				"time" : {
      					"$lte" : ISODate("2019-02-13T11:36:03.481Z")
      				}
      			}
      		}
      	}
      ]
      

      It appears as if the $match is getting pushed before the $project. The $cursor stage fetches the matching documents, and then the $_internalUnpackBucket excludes the time field, which represents the $project.

      For this query to be correct, a $match should not be pushed before a $project if the $project modifies a field included in the $match.

            Assignee:
            ivan.fefer@mongodb.com Ivan Fefer
            Reporter:
            davis.haupt@mongodb.com Davis Haupt (Inactive)
            Votes:
            0 Vote for this issue
            Watchers:
            6 Start watching this issue

              Created:
              Updated:
              Resolved: