OTel metrics file export helpers double-count a node after exporter file rotation

ExportXMLWordPrintableJSON

    • Type: Bug
    • Resolution: Unresolved
    • Priority: Major - P3
    • None
    • Affects Version/s: None
    • Component/s: None
    • DB Integration & Observability
    • ALL
    • Hide 
      import {getHistogramByAttributeAcrossFiles} from "jstests/noPassthrough/observability/libs/otel_metrics_file_export_helpers.js";

function makeRecord(metrics) {
    return {
        resourceMetrics: [
            {
                resource: {attributes: []},
                scopeMetrics: [{metrics: metrics, scope: {name: "mongodb"}}],
            },
        ],
    };
}

function makeHistogramMetric(name, dataPoints) {
    return {name, histogram: {aggregationTemporality: 2, dataPoints}};
}

const dir = MongoRunner.toRealPath("otel_rotation_repro_" + ObjectId().str);
assert(mkdir(dir));

const record = makeRecord([
    makeHistogramMetric("h", [
        {
            startTimeUnixNano: "1000000000000000",
            timeUnixNano: "1000000000000000000",
            count: "1",
            sum: 5,
            bucketCounts: ["0", "1", "0", "0"],
            explicitBounds: [0, 10, 100],
            attributes: [{key: "outcome", value: {stringValue: "success"}}],
        },
    ]),
]);

// Same node, two files: one per side of the midnight UTC rotation.
writeFile(dir + "/mongodb-21791-20261102-1-metrics.jsonl", JSON.stringify(record) + "\n");
writeFile(dir + "/mongodb-21791-20261103-1-metrics.jsonl", JSON.stringify(record) + "\n");

const result = getHistogramByAttributeAcrossFiles(dir, "h", "outcome", "success");
jsTest.log.info("getHistogramByAttributeAcrossFiles result", {result});
assert.eq(result.count, 1, "node was double-counted after file rotation", {result});
      Show
      import {getHistogramByAttributeAcrossFiles} from "jstests/noPassthrough/observability/libs/otel_metrics_file_export_helpers.js" ; function makeRecord(metrics) { return { resourceMetrics: [ { resource: {attributes: []}, scopeMetrics: [{metrics: metrics, scope: {name: "mongodb" }}], }, ], }; } function makeHistogramMetric(name, dataPoints) { return {name, histogram: {aggregationTemporality: 2, dataPoints}}; } const dir = MongoRunner.toRealPath( "otel_rotation_repro_" + ObjectId().str); assert(mkdir(dir)); const record = makeRecord([ makeHistogramMetric( "h" , [ { startTimeUnixNano: "1000000000000000" , timeUnixNano: "1000000000000000000" , count: "1" , sum: 5, bucketCounts: [ "0" , "1" , "0" , "0" ], explicitBounds: [0, 10, 100], attributes: [{key: "outcome" , value: {stringValue: "success" }}], }, ]), ]); // Same node, two files: one per side of the midnight UTC rotation. writeFile(dir + "/mongodb-21791-20261102-1-metrics.jsonl" , JSON.stringify(record) + "\n" ); writeFile(dir + "/mongodb-21791-20261103-1-metrics.jsonl" , JSON.stringify(record) + "\n" ); const result = getHistogramByAttributeAcrossFiles(dir, "h" , "outcome" , "success" ); jsTest.log.info( "getHistogramByAttributeAcrossFiles result" , {result}); assert.eq(result.count, 1, "node was double -counted after file rotation" , {result});
    • 0
    • None
    • None
    • None
    • None
    • None
    • None
    • None

      The helpers in jstests/noPassthrough/observability/libs/otel_metrics_file_export_helpers.js assume one metrics file per node. For example getHistogramByAttributeAcrossFiles() (and its wrapper assertHistogramCountAcrossFilesSoon()) sums the last record of every *-metrics.jsonl file in the metrics directory. However, the OTel file exporter rotates files for a single node:

      • at midnight UTC, because the file name embeds the date (mongodb-<pid>%Y%m%d%N-metrics.jsonl, see src/mongo/otel/metrics/metrics_initialization.cpp)
      • when a file reaches the 8MB chunk limit (the %N rotation index)

      Every rotated file retains its last cumulative snapshot, so after a rotation a single node's metrics are summed once per file and the aggregated values are inflated. Any test using these helpers can flake if its run crosses midnight UTC or triggers the chunk limit. Readers that keep their own per-file summing logic outside this library have hit this in the wild: BF-47068 failed after a run crossed midnight UTC, because the node had two files each reporting the same cumulative counter value of 1.

            Assignee:
            Unassigned
            Reporter:
            Ernesto Rodriguez Reina
            Votes:
            0 Vote for this issue
            Watchers:
            2 Start watching this issue

              Created:
              Updated: