Uploaded image for project: 'Documentation'
  1. Documentation
  2. DOCS-15781

[SERVER] Streaming with ID on time

    XMLWordPrintableJSON

Details

    • Task
    • Status: Backlog
    • Major - P3
    • Resolution: Unresolved
    • None
    • 6.3.0-rc0
    • manual, Server
    • None
    • Hide
      In some cases for timeseries (example below) optimizer may replace $group with $_internalStreamingGroup stage.

      In addition to all $group fields, it will also contain $monotonicIdFields field that lists all _id fields that are considered monotonic by the optimizer. If _id is a single expression, it will just contain a single element “_id”.

      Example of explain output with the new stage
      {
              "explainVersion" : "1",
              "stages" : [
                      {
                              "$cursor" : {
                                      "queryPlanner" : {
                                              "namespace" : "timeseries.system.buckets.Collection0",
                                              "indexFilterSet" : false,
                                              "parsedQuery" : {

                                              },
                                              "queryHash" : "17830885",
                                              "planCacheKey" : "17830885",
                                              "maxIndexedOrSolutionsReached" : false,
                                              "maxIndexedAndSolutionsReached" : false,
                                              "maxScansToExplodeReached" : false,
                                              "winningPlan" : {
                                                      "stage" : "COLLSCAN",
                                                      "direction" : "forward"
                                              },
                                              "rejectedPlans" : [ ]
                                      }
                              }
                      },
                      {
                              "$_internalUnpackBucket" : {
                                      "include" : [
                                              "price",
                                              "time",
                                              "symbol"
                                      ],
                                      "timeField" : "time",
                                      "metaField" : "symbol",
                                      "bucketMaxSpanSeconds" : 3600,
                                      "assumeNoMixedSchemaData" : true,
                                      "includeMinTimeAsMetadata" : true

                      "internalQueryFacetBufferSizeBytes" : 104857600,
                      "internalQueryFacetMaxOutputDocSizeBytes" : 104857600,
                      "internalLookupStageIntermediateDocumentMaxSizeBytes" : 104857600,
                      "internalDocumentSourceGroupMaxMemoryBytes" : 104857600,
                      "internalQueryMaxBlockingSortMemoryUsageBytes" : 104857600,
                      "internalQueryProhibitBlockingMergeOnMongoS" : 0,
                      "internalQueryMaxAddToSetBytes" : 104857600,
                      "internalDocumentSourceSetWindowFieldsMaxMemoryBytes" : 104857600,
                      "internalQueryFrameworkControl" : "tryBonsai"
              },
              "command" : {
                      "aggregate" : "system.buckets.Collection0",
                      "pipeline" : [
                              {
                                      "$_internalUnpackBucket" : {
                                              "timeField" : "time",
                                              "metaField" : "symbol",
                                              "bucketMaxSpanSeconds" : 3600,
                                              "assumeNoMixedSchemaData" : true,
                                      "includeMinTimeAsMetadata" : true [75/5353]
                              }
                      },
                      {
                              "$_internalBoundedSort" : {
                                      "sortKey" : {
                                              "time" : 1
                                      },
                                      "bound" : {
                                              "base" : "min",
                                              "offsetSeconds" : NumberLong(0)
                                      },
                                      "limit" : NumberLong(0)
                              }
                      },
                      {
                              "$_internalStreamingGroup" : {
                                      "_id" : {
                                              "symbol" : "$symbol",
                                              "time" : {
                                                      "$dateTrunc" : {
                                                              "date" : "$time",
                                                              "unit" : {
                                                                      "$const" : "minute"
                                                              },
                                                              "binSize" : {
                                                                      "$const" : 61
                                                              }
                                                      }
                                              }
                                      },
                                      "high" : {
                                              "$max" : "$price"
                                      },
                                      "low" : {
                                              "$min" : "$price"
                                      },
                                      "open" : {
                                              "$first" : "$price"
                                      },
                                      "close" : {
                                              "$last" : "$price"
                                      },
                                      "$monotonicIdFields" : [
                                              "time"
                                      ]
                              }
                      }
              ],
              "serverInfo" : {
                      "host" : "ip-10-122-10-70",
                      "port" : 27017,
                      "version" : "6.3.0-alpha-261-geac3840",
                      "gitVersion" : "eac384080001c4ba2dbbb72cb79550deb215a79a"
              },
              "serverParameters" : {
                      "internalQueryFacetBufferSizeBytes" : 104857600,
                      "internalQueryFacetMaxOutputDocSizeBytes" : 104857600,
                      "internalLookupStageIntermediateDocumentMaxSizeBytes" : 104857600,
                      "internalDocumentSourceGroupMaxMemoryBytes" : 104857600,
                      "internalQueryMaxBlockingSortMemoryUsageBytes" : 104857600,
                      "internalQueryProhibitBlockingMergeOnMongoS" : 0,
                      "internalQueryMaxAddToSetBytes" : 104857600,
                      "internalDocumentSourceSetWindowFieldsMaxMemoryBytes" : 104857600,
                      "internalQueryFrameworkControl" : "tryBonsai"
              },
              "command" : {
                      "aggregate" : "system.buckets.Collection0",
                      "pipeline" : [
                              {
                                      "$_internalUnpackBucket" : {
                                              "timeField" : "time",
                                              "metaField" : "symbol",
                                              "bucketMaxSpanSeconds" : 3600,
                                              "assumeNoMixedSchemaData" : true,
                                              "usesExtendedRange" : false
                                      }
                              },
                              {
                                      "$sort" : {
                                              "time" : 1
                                      }
                              },
                              {
                                      "$group" : {
                                              "_id" : {
                                                      "symbol" : "$symbol",
                                                      "time" : {
                                                              "$dateTrunc" : {
                                                                      "date" : "$time",
                                                                      "unit" : "minute",
                                                                      "binSize" : 61
                                                              }
                                                      }
                                              },
                                              "high" : {
                                                      "$max" : "$price"
                                              },
                                              "low" : {
                                                      "$min" : "$price"
                                              },
                                              "open" : {
                                                      "$first" : "$price"
                                              },
                                              "close" : {
                                                      "$last" : "$price"
                                              }
                                      }
                              }
                      ],
                      "cursor" : {

                      },
                      "collation" : {
                              "locale" : "simple"
                      }
              },
              "ok" : 1
      }
      Show
      In some cases for timeseries (example below) optimizer may replace $group with $_internalStreamingGroup stage. In addition to all $group fields, it will also contain $monotonicIdFields field that lists all _id fields that are considered monotonic by the optimizer. If _id is a single expression, it will just contain a single element “_id”. Example of explain output with the new stage {         "explainVersion" : "1",         "stages" : [                 {                         "$cursor" : {                                 "queryPlanner" : {                                         "namespace" : "timeseries.system.buckets.Collection0",                                         "indexFilterSet" : false,                                         "parsedQuery" : {                                         },                                         "queryHash" : "17830885",                                         "planCacheKey" : "17830885",                                         "maxIndexedOrSolutionsReached" : false,                                         "maxIndexedAndSolutionsReached" : false,                                         "maxScansToExplodeReached" : false,                                         "winningPlan" : {                                                 "stage" : "COLLSCAN",                                                 "direction" : "forward"                                         },                                         "rejectedPlans" : [ ]                                 }                         }                 },                 {                         "$_internalUnpackBucket" : {                                 "include" : [                                         "price",                                         "time",                                         "symbol"                                 ],                                 "timeField" : "time",                                 "metaField" : "symbol",                                 "bucketMaxSpanSeconds" : 3600,                                 "assumeNoMixedSchemaData" : true,                                 "includeMinTimeAsMetadata" : true                 "internalQueryFacetBufferSizeBytes" : 104857600,                 "internalQueryFacetMaxOutputDocSizeBytes" : 104857600,                 "internalLookupStageIntermediateDocumentMaxSizeBytes" : 104857600,                 "internalDocumentSourceGroupMaxMemoryBytes" : 104857600,                 "internalQueryMaxBlockingSortMemoryUsageBytes" : 104857600,                 "internalQueryProhibitBlockingMergeOnMongoS" : 0,                 "internalQueryMaxAddToSetBytes" : 104857600,                 "internalDocumentSourceSetWindowFieldsMaxMemoryBytes" : 104857600,                 "internalQueryFrameworkControl" : "tryBonsai"         },         "command" : {                 "aggregate" : "system.buckets.Collection0",                 "pipeline" : [                         {                                 "$_internalUnpackBucket" : {                                         "timeField" : "time",                                         "metaField" : "symbol",                                         "bucketMaxSpanSeconds" : 3600,                                         "assumeNoMixedSchemaData" : true,                                 "includeMinTimeAsMetadata" : true [75/5353]                         }                 },                 {                         "$_internalBoundedSort" : {                                 "sortKey" : {                                         "time" : 1                                 },                                 "bound" : {                                         "base" : "min",                                         "offsetSeconds" : NumberLong(0)                                 },                                 "limit" : NumberLong(0)                         }                 },                 {                         "$_internalStreamingGroup" : {                                 "_id" : {                                         "symbol" : "$symbol",                                         "time" : {                                                 "$dateTrunc" : {                                                         "date" : "$time",                                                         "unit" : {                                                                 "$const" : "minute"                                                         },                                                         "binSize" : {                                                                 "$const" : 61                                                         }                                                 }                                         }                                 },                                 "high" : {                                         "$max" : "$price"                                 },                                 "low" : {                                         "$min" : "$price"                                 },                                 "open" : {                                         "$first" : "$price"                                 },                                 "close" : {                                         "$last" : "$price"                                 },                                 "$monotonicIdFields" : [                                         "time"                                 ]                         }                 }         ],         "serverInfo" : {                 "host" : "ip-10-122-10-70",                 "port" : 27017,                 "version" : "6.3.0-alpha-261-geac3840",                 "gitVersion" : "eac384080001c4ba2dbbb72cb79550deb215a79a"         },         "serverParameters" : {                 "internalQueryFacetBufferSizeBytes" : 104857600,                 "internalQueryFacetMaxOutputDocSizeBytes" : 104857600,                 "internalLookupStageIntermediateDocumentMaxSizeBytes" : 104857600,                 "internalDocumentSourceGroupMaxMemoryBytes" : 104857600,                 "internalQueryMaxBlockingSortMemoryUsageBytes" : 104857600,                 "internalQueryProhibitBlockingMergeOnMongoS" : 0,                 "internalQueryMaxAddToSetBytes" : 104857600,                 "internalDocumentSourceSetWindowFieldsMaxMemoryBytes" : 104857600,                 "internalQueryFrameworkControl" : "tryBonsai"         },         "command" : {                 "aggregate" : "system.buckets.Collection0",                 "pipeline" : [                         {                                 "$_internalUnpackBucket" : {                                         "timeField" : "time",                                         "metaField" : "symbol",                                         "bucketMaxSpanSeconds" : 3600,                                         "assumeNoMixedSchemaData" : true,                                         "usesExtendedRange" : false                                 }                         },                         {                                 "$sort" : {                                         "time" : 1                                 }                         },                         {                                 "$group" : {                                         "_id" : {                                                 "symbol" : "$symbol",                                                 "time" : {                                                         "$dateTrunc" : {                                                                 "date" : "$time",                                                                 "unit" : "minute",                                                                 "binSize" : 61                                                         }                                                 }                                         },                                         "high" : {                                                 "$max" : "$price"                                         },                                         "low" : {                                                 "$min" : "$price"                                         },                                         "open" : {                                                 "$first" : "$price"                                         },                                         "close" : {                                                 "$last" : "$price"                                         }                                 }                         }                 ],                 "cursor" : {                 },                 "collation" : {                         "locale" : "simple"                 }         },         "ok" : 1 }
    • true

    Description

      Original title:
      Investigate changes in SERVER-70267: Streaming group when id is on time

      Original Downstream Change Summary

      In some cases for timeseries (example below) optimizer may replace $group with $_internalStreamingGroup stage.

      In addition to all $group fields, it will also contain $monotonicIdFields field that lists all _id fields that are considered monotonic by the optimizer. If _id is a single expression, it will just contain a single element “_id”.

      Example of explain output with the new stage
      {
      "explainVersion" : "1",
      "stages" : [
      {
      "$cursor" : {
      "queryPlanner" : {
      "namespace" : "timeseries.system.buckets.Collection0",
      "indexFilterSet" : false,
      "parsedQuery" : {

      },
      "queryHash" : "17830885",
      "planCacheKey" : "17830885",
      "maxIndexedOrSolutionsReached" : false,
      "maxIndexedAndSolutionsReached" : false,
      "maxScansToExplodeReached" : false,
      "winningPlan" :

      { "stage" : "COLLSCAN", "direction" : "forward" }

      ,
      "rejectedPlans" : [ ]
      }
      }
      },
      {
      "$_internalUnpackBucket" :

      { "include" : [ "price", "time", "symbol" ], "timeField" : "time", "metaField" : "symbol", "bucketMaxSpanSeconds" : 3600, "assumeNoMixedSchemaData" : true, "includeMinTimeAsMetadata" : true "internalQueryFacetBufferSizeBytes" : 104857600, "internalQueryFacetMaxOutputDocSizeBytes" : 104857600, "internalLookupStageIntermediateDocumentMaxSizeBytes" : 104857600, "internalDocumentSourceGroupMaxMemoryBytes" : 104857600, "internalQueryMaxBlockingSortMemoryUsageBytes" : 104857600, "internalQueryProhibitBlockingMergeOnMongoS" : 0, "internalQueryMaxAddToSetBytes" : 104857600, "internalDocumentSourceSetWindowFieldsMaxMemoryBytes" : 104857600, "internalQueryFrameworkControl" : "tryBonsai" }

      ,
      "command" : {
      "aggregate" : "system.buckets.Collection0",
      "pipeline" : [
      {
      "$_internalUnpackBucket" :

      { "timeField" : "time", "metaField" : "symbol", "bucketMaxSpanSeconds" : 3600, "assumeNoMixedSchemaData" : true, "includeMinTimeAsMetadata" : true [75/5353] }

      },
      {
      "$_internalBoundedSort" : {
      "sortKey" :

      { "time" : 1 }

      ,
      "bound" :

      { "base" : "min", "offsetSeconds" : NumberLong(0) }

      ,
      "limit" : NumberLong(0)
      }
      },
      {
      "$_internalStreamingGroup" : {
      "_id" : {
      "symbol" : "$symbol",
      "time" : {
      "$dateTrunc" : {
      "date" : "$time",
      "unit" :

      { "$const" : "minute" }

      ,
      "binSize" :

      { "$const" : 61 }

      }
      }
      },
      "high" :

      { "$max" : "$price" }

      ,
      "low" :

      { "$min" : "$price" }

      ,
      "open" :

      { "$first" : "$price" }

      ,
      "close" :

      { "$last" : "$price" }

      ,
      "$monotonicIdFields" : [
      "time"
      ]
      }
      }
      ],
      "serverInfo" :

      { "host" : "ip-10-122-10-70", "port" : 27017, "version" : "6.3.0-alpha-261-geac3840", "gitVersion" : "eac384080001c4ba2dbbb72cb79550deb215a79a" }

      ,
      "serverParameters" :

      { "internalQueryFacetBufferSizeBytes" : 104857600, "internalQueryFacetMaxOutputDocSizeBytes" : 104857600, "internalLookupStageIntermediateDocumentMaxSizeBytes" : 104857600, "internalDocumentSourceGroupMaxMemoryBytes" : 104857600, "internalQueryMaxBlockingSortMemoryUsageBytes" : 104857600, "internalQueryProhibitBlockingMergeOnMongoS" : 0, "internalQueryMaxAddToSetBytes" : 104857600, "internalDocumentSourceSetWindowFieldsMaxMemoryBytes" : 104857600, "internalQueryFrameworkControl" : "tryBonsai" }

      ,
      "command" : {
      "aggregate" : "system.buckets.Collection0",
      "pipeline" : [
      {
      "$_internalUnpackBucket" :

      { "timeField" : "time", "metaField" : "symbol", "bucketMaxSpanSeconds" : 3600, "assumeNoMixedSchemaData" : true, "usesExtendedRange" : false }

      },
      {
      "$sort" :

      { "time" : 1 }

      },
      {
      "$group" : {
      "_id" : {
      "symbol" : "$symbol",
      "time" : {
      "$dateTrunc" :

      { "date" : "$time", "unit" : "minute", "binSize" : 61 }

      }
      },
      "high" :

      { "$max" : "$price" }

      ,
      "low" :

      { "$min" : "$price" }

      ,
      "open" :

      { "$first" : "$price" }

      ,
      "close" :

      { "$last" : "$price" }

      }
      }
      ],
      "cursor" : {

      },
      "collation" :

      { "locale" : "simple" }

      },
      "ok" : 1
      }

      Description of Linked Ticket

      null

      Attachments

        Issue Links

          Activity

            People

              Unassigned Unassigned
              backlog-server-pm Backlog - Core Eng Program Management Team
              Sarah Olson Sarah Olson
              Votes:
              0 Vote for this issue
              Watchers:
              3 Start watching this issue

              Dates

                Created:
                Updated:
                6 days ago