diff --git a/CHANGELOG.md b/CHANGELOG.md index 30f5fa0797a..ff9e7746b59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ For notes on migrating to 2.x / 0.200.x see [the upgrade guide](doc/upgrade-to-2 ### :rocket: Features +* feat(sdk-metrics): add maxExportBatchSize option to PeriodicExportingMetricReader [#6655](https://github.com/open-telemetry/opentelemetry-js/pull/6655) @psx95 + * feat(sdk-trace-base): pretty-print `SpanImpl`, `Tracer`, and `BasicTracerProvider` via `util.inspect` so they render through `diag` and `console.log` [#6690](https://github.com/open-telemetry/opentelemetry-js/pull/6690) @mcollina ### :bug: Bug Fixes diff --git a/packages/sdk-metrics/src/export/MetricDataSplitter.ts b/packages/sdk-metrics/src/export/MetricDataSplitter.ts new file mode 100644 index 00000000000..6a2a6f2f787 --- /dev/null +++ b/packages/sdk-metrics/src/export/MetricDataSplitter.ts @@ -0,0 +1,92 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { DataPoint, ResourceMetrics, ScopeMetrics } from './MetricData'; + +/** + * Splits a ResourceMetrics object into smaller ResourceMetrics objects + * such that no batch exceeds maxExportBatchSize data points. + * @param resourceMetrics The metrics to split. + * @param maxExportBatchSize The maximum number of data points per batch. + * @internal + */ +export function splitMetricData( + resourceMetrics: ResourceMetrics, + maxExportBatchSize: number +): ResourceMetrics[] { + if (!Number.isInteger(maxExportBatchSize) || maxExportBatchSize <= 0) { + throw new Error('maxExportBatchSize must be a positive integer'); + } + const batches: ResourceMetrics[] = []; + let currentBatchPoints = 0; + let currentScopeMetrics: ScopeMetrics[] = []; + + function flush() { + if (currentScopeMetrics.length > 0) { + batches.push({ + resource: resourceMetrics.resource, + scopeMetrics: currentScopeMetrics, + }); + currentScopeMetrics = []; + currentBatchPoints = 0; + } + } + + // Iterate through all scopes in the input metrics + for (const scopeMetric of resourceMetrics.scopeMetrics) { + let scopeMetricCopy: ScopeMetrics | null = null; + + // Iterate through all metrics within the current scope + for (const metric of scopeMetric.metrics) { + let dataPointsRemaining = metric.dataPoints; + let metricCopy: typeof metric | undefined = undefined; + + // If a metric has no data points, add it directly to the current batch + if (dataPointsRemaining.length === 0) { + if (!scopeMetricCopy) { + scopeMetricCopy = { scope: scopeMetric.scope, metrics: [] }; + currentScopeMetrics.push(scopeMetricCopy); + } + scopeMetricCopy.metrics.push(metric); + continue; + } + + // Chunk the data points of the current metric across batches + while (dataPointsRemaining.length > 0) { + const spaceLeft = maxExportBatchSize - currentBatchPoints; + const take = Math.min(spaceLeft, dataPointsRemaining.length); + const chunk = dataPointsRemaining.slice(0, take); + dataPointsRemaining = dataPointsRemaining.slice(take); + + // Ensure we have a ScopeMetrics object in the current batch + if (!scopeMetricCopy) { + scopeMetricCopy = { scope: scopeMetric.scope, metrics: [] }; + currentScopeMetrics.push(scopeMetricCopy); + metricCopy = undefined; // Reset because we are starting a new batch + } + + // Ensure we have a MetricData object for this specific metric in the current batch. + if (!metricCopy) { + metricCopy = { ...metric, dataPoints: [] }; + scopeMetricCopy.metrics.push(metricCopy); + } + + (metricCopy.dataPoints as DataPoint[]).push( + ...(chunk as DataPoint[]) + ); + currentBatchPoints += take; + + // If the current batch is full, flush it and start a new one + if (currentBatchPoints === maxExportBatchSize) { + flush(); + scopeMetricCopy = null; // Force recreation of scope copy in the next batch + } + } + } + } + + flush(); + return batches; +} diff --git a/packages/sdk-metrics/src/export/PeriodicExportingMetricReader.ts b/packages/sdk-metrics/src/export/PeriodicExportingMetricReader.ts index df489066883..04ec871063d 100644 --- a/packages/sdk-metrics/src/export/PeriodicExportingMetricReader.ts +++ b/packages/sdk-metrics/src/export/PeriodicExportingMetricReader.ts @@ -14,6 +14,7 @@ import type { PushMetricExporter } from './MetricExporter'; import { callWithTimeout, TimeoutError } from '../utils'; import type { MetricProducer } from './MetricProducer'; import { InstrumentType } from './MetricData'; +import { splitMetricData } from './MetricDataSplitter'; export type PeriodicExportingMetricReaderOptions = { /** @@ -50,6 +51,12 @@ export type PeriodicExportingMetricReaderOptions = { observableUpDownCounter?: number; default?: number; }; + /** + * The maximum batch size for exports. If configured, the reader will split + * batches larger than this size into smaller batches. + * @experimental + */ + maxExportBatchSize?: number; }; /** @@ -61,6 +68,8 @@ export class PeriodicExportingMetricReader extends MetricReader { private _exporter: PushMetricExporter; private readonly _exportInterval: number; private readonly _exportTimeout: number; + private readonly _maxExportBatchSize?: number; + private _ongoingExportPromise: Promise | null = null; constructor(options: PeriodicExportingMetricReaderOptions) { const { @@ -68,6 +77,7 @@ export class PeriodicExportingMetricReader extends MetricReader { exportIntervalMillis = 60000, metricProducers, cardinalityLimits, + maxExportBatchSize, } = options; let { exportTimeoutMillis = 30000 } = options; @@ -111,6 +121,13 @@ export class PeriodicExportingMetricReader extends MetricReader { throw Error('exportTimeoutMillis must be greater than 0'); } + if ( + maxExportBatchSize !== undefined && + (!Number.isInteger(maxExportBatchSize) || maxExportBatchSize <= 0) + ) { + throw Error('maxExportBatchSize must be a positive integer'); + } + if (exportIntervalMillis < exportTimeoutMillis) { if ( 'exportIntervalMillis' in options && @@ -132,25 +149,25 @@ export class PeriodicExportingMetricReader extends MetricReader { this._exportInterval = exportIntervalMillis; this._exportTimeout = exportTimeoutMillis; this._exporter = exporter; + this._maxExportBatchSize = maxExportBatchSize; } private async _runOnce(): Promise { try { - await callWithTimeout(this._doRun(), this._exportTimeout); + await this._doRun(); } catch (err) { - if (err instanceof TimeoutError) { - api.diag.error( - 'Export took longer than %s milliseconds and timed out.', - this._exportTimeout - ); - return; - } - globalErrorHandler(err); } } private async _doRun(): Promise { + if (this._ongoingExportPromise) { + api.diag.debug( + 'PeriodicExportingMetricReader: export already in progress, skipping' + ); + return; + } + const { resourceMetrics, errors } = await this.collect({ timeoutMillis: this._exportTimeout, }); @@ -175,11 +192,51 @@ export class PeriodicExportingMetricReader extends MetricReader { return; } - const result = await internal._export(this._exporter, resourceMetrics); - if (result.code !== ExportResultCode.SUCCESS) { - throw new Error( - `PeriodicExportingMetricReader: metrics export failed (error ${result.error})` - ); + const batches = this._maxExportBatchSize + ? splitMetricData(resourceMetrics, this._maxExportBatchSize) + : [resourceMetrics]; + + const currentExport = async () => { + let anyErr: Error | null = null; + for (const batch of batches) { + try { + const result = await callWithTimeout( + internal._export(this._exporter, batch), + this._exportTimeout + ); + if (result.code !== ExportResultCode.SUCCESS) { + const err = new Error( + `PeriodicExportingMetricReader: metrics export failed (error ${result.error})` + ); + api.diag.error(err.message); + anyErr = err; + } + } catch (e) { + if (e instanceof TimeoutError) { + // We do not report TimeoutError to the globalErrorHandler in _runOnce(). + api.diag.error( + `PeriodicExportingMetricReader: metrics export timed out after ${this._exportTimeout}ms` + ); + break; + } else { + api.diag.error( + 'PeriodicExportingMetricReader: metrics export threw error', + e + ); + anyErr = e instanceof Error ? e : new Error(String(e)); + } + } + } + if (anyErr) { + throw anyErr; + } + }; + + this._ongoingExportPromise = currentExport(); + try { + await this._ongoingExportPromise; + } finally { + this._ongoingExportPromise = null; } } diff --git a/packages/sdk-metrics/test/export/MetricDataSplitter.test.ts b/packages/sdk-metrics/test/export/MetricDataSplitter.test.ts new file mode 100644 index 00000000000..8f0210825cf --- /dev/null +++ b/packages/sdk-metrics/test/export/MetricDataSplitter.test.ts @@ -0,0 +1,1007 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { splitMetricData } from '../../src/export/MetricDataSplitter'; +import type { ResourceMetrics } from '../../src/export/MetricData'; +import { DataPointType } from '../../src/export/MetricData'; +import { AggregationTemporality } from '../../src/export/AggregationTemporality'; +import { ValueType } from '@opentelemetry/api'; +import * as assert from 'assert'; + +describe('splitMetricData', () => { + const dummyResource = { attributes: {} } as any; + + describe('GAUGE', () => { + it('should split batches when exceeding maxExportBatchSize (GAUGE)', () => { + const resourceMetrics: ResourceMetrics = { + resource: dummyResource, + scopeMetrics: [ + { + scope: { name: 'test' }, + metrics: [ + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 1, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 2, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 3, + }, + ], + descriptor: { + name: 'm1', + description: 'desc1', + unit: 'unit1', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + ], + }; + + const batches = splitMetricData(resourceMetrics, 2); + + assert.strictEqual(batches.length, 2); + + // First batch should have 2 data points + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].dataPoints.length, + 2 + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].dataPoints[0].value, + 1 + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].dataPoints[1].value, + 2 + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].descriptor.name, + 'm1' + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].descriptor.description, + 'desc1' + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].descriptor.unit, + 'unit1' + ); + + // Second batch should have 1 data point + assert.strictEqual( + batches[1].scopeMetrics[0].metrics[0].dataPoints.length, + 1 + ); + assert.strictEqual( + batches[1].scopeMetrics[0].metrics[0].dataPoints[0].value, + 3 + ); + assert.strictEqual( + batches[1].scopeMetrics[0].metrics[0].descriptor.name, + 'm1' + ); + assert.strictEqual( + batches[1].scopeMetrics[0].metrics[0].descriptor.description, + 'desc1' + ); + assert.strictEqual( + batches[1].scopeMetrics[0].metrics[0].descriptor.unit, + 'unit1' + ); + }); + + it('should split data points across metrics if needed (GAUGE)', () => { + const resourceMetrics: ResourceMetrics = { + resource: dummyResource, + scopeMetrics: [ + { + scope: { name: 'test' }, + metrics: [ + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 1, + }, + ], + descriptor: { + name: 'm1', + description: 'desc1', + unit: 'unit1', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 2, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 3, + }, + ], + descriptor: { + name: 'm2', + description: 'desc2', + unit: 'unit2', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + ], + }; + + const batches = splitMetricData(resourceMetrics, 2); + + assert.strictEqual(batches.length, 2); + + // First batch should have 2 data points (m1:1, m2:2) + assert.strictEqual(batches[0].scopeMetrics[0].metrics.length, 2); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].dataPoints.length, + 1 + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].dataPoints[0].value, + 1 + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].descriptor.name, + 'm1' + ); + + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[1].dataPoints.length, + 1 + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[1].dataPoints[0].value, + 2 + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[1].descriptor.name, + 'm2' + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[1].descriptor.description, + 'desc2' + ); + + // Second batch should have 1 data point (m2:3) + assert.strictEqual(batches[1].scopeMetrics[0].metrics.length, 1); + assert.strictEqual( + batches[1].scopeMetrics[0].metrics[0].descriptor.name, + 'm2' + ); + assert.strictEqual( + batches[1].scopeMetrics[0].metrics[0].descriptor.description, + 'desc2' + ); + assert.strictEqual( + batches[1].scopeMetrics[0].metrics[0].dataPoints.length, + 1 + ); + assert.strictEqual( + batches[1].scopeMetrics[0].metrics[0].dataPoints[0].value, + 3 + ); + }); + + it('should handle empty data points (GAUGE)', () => { + const resourceMetrics: ResourceMetrics = { + resource: dummyResource, + scopeMetrics: [ + { + scope: { name: 'test' }, + metrics: [ + { + dataPointType: DataPointType.GAUGE, + dataPoints: [], + descriptor: { + name: 'm1', + description: 'desc1', + unit: 'unit1', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + ], + }; + + const batches = splitMetricData(resourceMetrics, 2); + + assert.strictEqual(batches.length, 1); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].dataPoints.length, + 0 + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].descriptor.name, + 'm1' + ); + }); + + it('should split across multiple scopes and fill batches (GAUGE)', () => { + const resourceMetrics: ResourceMetrics = { + resource: dummyResource, + scopeMetrics: [ + { + scope: { name: 'scope1' }, + metrics: [ + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 1, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 2, + }, + ], + descriptor: { + name: 'm1', + description: 'desc1', + unit: 'unit1', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 3, + }, + ], + descriptor: { + name: 'm2', + description: 'desc2', + unit: 'unit2', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + { + scope: { name: 'scope2' }, + metrics: [ + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 4, + }, + ], + descriptor: { + name: 'm3', + description: 'desc3', + unit: 'unit3', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + ], + }; + + const batches = splitMetricData(resourceMetrics, 2); + + assert.strictEqual(batches.length, 2); + + // Batch 1: scope1, m1 (2 points) + assert.strictEqual(batches[0].scopeMetrics.length, 1); + assert.strictEqual(batches[0].scopeMetrics[0].scope.name, 'scope1'); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].descriptor.name, + 'm1' + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].dataPoints.length, + 2 + ); + + // Batch 2: scope1, m2 (1 point) AND scope2, m3 (1 point) + assert.strictEqual(batches[1].scopeMetrics.length, 2); + assert.strictEqual(batches[1].scopeMetrics[0].scope.name, 'scope1'); + assert.strictEqual( + batches[1].scopeMetrics[0].metrics[0].descriptor.name, + 'm2' + ); + assert.strictEqual( + batches[1].scopeMetrics[0].metrics[0].descriptor.description, + 'desc2' + ); + assert.strictEqual( + batches[1].scopeMetrics[0].metrics[0].dataPoints.length, + 1 + ); + + assert.strictEqual(batches[1].scopeMetrics[1].scope.name, 'scope2'); + assert.strictEqual( + batches[1].scopeMetrics[1].metrics[0].descriptor.name, + 'm3' + ); + assert.strictEqual( + batches[1].scopeMetrics[1].metrics[0].descriptor.unit, + 'unit3' + ); + assert.strictEqual( + batches[1].scopeMetrics[1].metrics[0].dataPoints.length, + 1 + ); + }); + + it('should split a single metric across multiple batches (GAUGE)', () => { + const resourceMetrics: ResourceMetrics = { + resource: dummyResource, + scopeMetrics: [ + { + scope: { name: 'test' }, + metrics: [ + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 1, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 2, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 3, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 4, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 5, + }, + ], + descriptor: { + name: 'm1', + description: 'desc1', + unit: 'unit1', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + ], + }; + + const batches = splitMetricData(resourceMetrics, 2); + + assert.strictEqual(batches.length, 3); + + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].dataPoints.length, + 2 + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].descriptor.name, + 'm1' + ); + + assert.strictEqual( + batches[1].scopeMetrics[0].metrics[0].dataPoints.length, + 2 + ); + assert.strictEqual( + batches[1].scopeMetrics[0].metrics[0].descriptor.name, + 'm1' + ); + + assert.strictEqual( + batches[2].scopeMetrics[0].metrics[0].dataPoints.length, + 1 + ); + assert.strictEqual( + batches[2].scopeMetrics[0].metrics[0].descriptor.name, + 'm1' + ); + }); + + it('should handle partly filled batches with multiple scopes (GAUGE)', () => { + const resourceMetrics: ResourceMetrics = { + resource: dummyResource, + scopeMetrics: [ + { + scope: { name: 'scope1' }, + metrics: [ + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 1, + }, + ], + descriptor: { + name: 'm1', + description: 'desc1', + unit: 'unit1', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + { + scope: { name: 'scope2' }, + metrics: [ + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 2, + }, + ], + descriptor: { + name: 'm2', + description: 'desc2', + unit: 'unit2', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + ], + }; + + const batches = splitMetricData(resourceMetrics, 3); + + assert.strictEqual(batches.length, 1); + assert.strictEqual(batches[0].scopeMetrics.length, 2); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].descriptor.name, + 'm1' + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].dataPoints.length, + 1 + ); + assert.strictEqual( + batches[0].scopeMetrics[1].metrics[0].descriptor.name, + 'm2' + ); + assert.strictEqual( + batches[0].scopeMetrics[1].metrics[0].dataPoints.length, + 1 + ); + }); + }); + + describe('SUM', () => { + it('should split batches when exceeding maxExportBatchSize (SUM)', () => { + const resourceMetrics: ResourceMetrics = { + resource: dummyResource, + scopeMetrics: [ + { + scope: { name: 'test' }, + metrics: [ + { + dataPointType: DataPointType.SUM, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 1, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 2, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 3, + }, + ], + descriptor: { + name: 'm1', + description: 'desc1', + unit: 'unit1', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + isMonotonic: true, + }, + ], + }, + ], + }; + + const batches = splitMetricData(resourceMetrics, 2); + + assert.strictEqual(batches.length, 2); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].dataPoints.length, + 2 + ); + assert.strictEqual( + (batches[0].scopeMetrics[0].metrics[0] as any).isMonotonic, + true + ); + assert.strictEqual( + batches[1].scopeMetrics[0].metrics[0].dataPoints.length, + 1 + ); + }); + + it('should split across multiple scopes and fill batches (SUM)', () => { + const resourceMetrics: ResourceMetrics = { + resource: dummyResource, + scopeMetrics: [ + { + scope: { name: 'scope1' }, + metrics: [ + { + dataPointType: DataPointType.SUM, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 1, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 2, + }, + ], + descriptor: { + name: 'm1', + description: 'desc1', + unit: 'unit1', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + isMonotonic: true, + }, + { + dataPointType: DataPointType.SUM, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 3, + }, + ], + descriptor: { + name: 'm2', + description: 'desc2', + unit: 'unit2', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + isMonotonic: true, + }, + ], + }, + { + scope: { name: 'scope2' }, + metrics: [ + { + dataPointType: DataPointType.SUM, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 4, + }, + ], + descriptor: { + name: 'm3', + description: 'desc3', + unit: 'unit3', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + isMonotonic: true, + }, + ], + }, + ], + }; + + const batches = splitMetricData(resourceMetrics, 2); + + assert.strictEqual(batches.length, 2); + assert.strictEqual(batches[0].scopeMetrics.length, 1); + assert.strictEqual(batches[1].scopeMetrics.length, 2); + }); + }); + + describe('HISTOGRAM', () => { + it('should split batches when exceeding maxExportBatchSize (HISTOGRAM)', () => { + const dummyHistogram = { + buckets: { boundaries: [1, 2], counts: [1, 1, 1] }, + sum: 3, + count: 3, + }; + const resourceMetrics: ResourceMetrics = { + resource: dummyResource, + scopeMetrics: [ + { + scope: { name: 'test' }, + metrics: [ + { + dataPointType: DataPointType.HISTOGRAM, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: dummyHistogram, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: dummyHistogram, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: dummyHistogram, + }, + ], + descriptor: { + name: 'm1', + description: 'desc1', + unit: 'unit1', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + ], + }; + + const batches = splitMetricData(resourceMetrics, 2); + + assert.strictEqual(batches.length, 2); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].dataPoints.length, + 2 + ); + assert.strictEqual( + batches[1].scopeMetrics[0].metrics[0].dataPoints.length, + 1 + ); + }); + + it('should split across multiple scopes and fill batches (HISTOGRAM)', () => { + const dummyHistogram = { + buckets: { boundaries: [1, 2], counts: [1, 1, 1] }, + sum: 3, + count: 3, + }; + const resourceMetrics: ResourceMetrics = { + resource: dummyResource, + scopeMetrics: [ + { + scope: { name: 'scope1' }, + metrics: [ + { + dataPointType: DataPointType.HISTOGRAM, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: dummyHistogram, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: dummyHistogram, + }, + ], + descriptor: { + name: 'm1', + description: 'desc1', + unit: 'unit1', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + { + dataPointType: DataPointType.HISTOGRAM, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: dummyHistogram, + }, + ], + descriptor: { + name: 'm2', + description: 'desc2', + unit: 'unit2', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + { + scope: { name: 'scope2' }, + metrics: [ + { + dataPointType: DataPointType.HISTOGRAM, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: dummyHistogram, + }, + ], + descriptor: { + name: 'm3', + description: 'desc3', + unit: 'unit3', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + ], + }; + + const batches = splitMetricData(resourceMetrics, 2); + + assert.strictEqual(batches.length, 2); + assert.strictEqual(batches[0].scopeMetrics.length, 1); + assert.strictEqual(batches[1].scopeMetrics.length, 2); + }); + }); + + describe('EXPONENTIAL_HISTOGRAM', () => { + it('should split batches when exceeding maxExportBatchSize (EXPONENTIAL_HISTOGRAM)', () => { + const dummyExponentialHistogram = { + sum: 3, + count: 3, + scale: 1, + zeroCount: 0, + positive: { offset: 1, bucketCounts: [1, 1, 1] }, + negative: { offset: 1, bucketCounts: [] }, + }; + const resourceMetrics: ResourceMetrics = { + resource: dummyResource, + scopeMetrics: [ + { + scope: { name: 'test' }, + metrics: [ + { + dataPointType: DataPointType.EXPONENTIAL_HISTOGRAM, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: dummyExponentialHistogram, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: dummyExponentialHistogram, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: dummyExponentialHistogram, + }, + ], + descriptor: { + name: 'm1', + description: 'desc1', + unit: 'unit1', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + ], + }; + + const batches = splitMetricData(resourceMetrics, 2); + + assert.strictEqual(batches.length, 2); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].dataPoints.length, + 2 + ); + assert.strictEqual( + batches[1].scopeMetrics[0].metrics[0].dataPoints.length, + 1 + ); + }); + + it('should not merge metrics with the same name but different types', () => { + const resourceMetrics: ResourceMetrics = { + resource: dummyResource, + scopeMetrics: [ + { + scope: { name: 'test' }, + metrics: [ + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 1, + }, + ], + descriptor: { + name: 'm1', + description: 'desc1', + unit: 'unit1', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + { + dataPointType: DataPointType.SUM, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 2, + }, + ], + descriptor: { + name: 'm1', // Same name! + description: 'desc2', + unit: 'unit2', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + isMonotonic: true, + }, + ], + }, + ], + }; + + const batches = splitMetricData(resourceMetrics, 10); + + assert.strictEqual(batches.length, 1); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics.length, + 2, + 'Should keep distinct metrics separate' + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].descriptor.name, + 'm1' + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[1].descriptor.name, + 'm1' + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[0].dataPointType, + DataPointType.GAUGE + ); + assert.strictEqual( + batches[0].scopeMetrics[0].metrics[1].dataPointType, + DataPointType.SUM + ); + }); + }); + + describe('Validation', () => { + it('should throw when maxExportBatchSize is less than or equal to 0', () => { + const resourceMetrics: ResourceMetrics = { + resource: dummyResource, + scopeMetrics: [], + }; + + assert.throws( + () => splitMetricData(resourceMetrics, 0), + /maxExportBatchSize must be a positive integer/ + ); + + assert.throws( + () => splitMetricData(resourceMetrics, -1), + /maxExportBatchSize must be a positive integer/ + ); + + assert.throws( + () => splitMetricData(resourceMetrics, -1.5), + /maxExportBatchSize must be a positive integer/ + ); + }); + }); +}); diff --git a/packages/sdk-metrics/test/export/PeriodicExportingMetricReader.test.ts b/packages/sdk-metrics/test/export/PeriodicExportingMetricReader.test.ts index 35e19b0ece1..9433639ad2a 100644 --- a/packages/sdk-metrics/test/export/PeriodicExportingMetricReader.test.ts +++ b/packages/sdk-metrics/test/export/PeriodicExportingMetricReader.test.ts @@ -32,7 +32,7 @@ import { DEFAULT_AGGREGATION_SELECTOR, DEFAULT_AGGREGATION_TEMPORALITY_SELECTOR, } from '../../src/export/AggregationSelector'; -import { ValueType } from '@opentelemetry/api'; +import { ValueType, diag } from '@opentelemetry/api'; const MAX_32_BIT_INT = 2 ** 31 - 1; @@ -42,6 +42,8 @@ class TestMetricExporter implements PushMetricExporter { public throwExport = false; public throwFlush = false; public rejectExport = false; + public concurrentCalls = 0; + public maxConcurrentCalls = 0; private _batches: ResourceMetrics[] = []; private _shutdown: boolean = false; @@ -50,11 +52,18 @@ class TestMetricExporter implements PushMetricExporter { resultCallback: (result: ExportResult) => void ): void { this._batches.push(metrics); + this.concurrentCalls++; + this.maxConcurrentCalls = Math.max( + this.maxConcurrentCalls, + this.concurrentCalls + ); if (this.throwExport) { + this.concurrentCalls--; throw new Error('Error during export'); } setTimeout(() => { + this.concurrentCalls--; if (this.rejectExport) { resultCallback({ code: ExportResultCode.FAILED, @@ -201,6 +210,42 @@ describe('PeriodicExportingMetricReader', () => { ); }); + it('should throw when maxExportBatchSize less or equal to 0', () => { + const exporter = new TestDeltaMetricExporter(); + assert.throws( + () => + new PeriodicExportingMetricReader({ + exporter: exporter, + exportIntervalMillis: 1, + exportTimeoutMillis: 1, + maxExportBatchSize: 0, + }), + /maxExportBatchSize must be a positive integer/ + ); + + assert.throws( + () => + new PeriodicExportingMetricReader({ + exporter: exporter, + exportIntervalMillis: 1, + exportTimeoutMillis: 1, + maxExportBatchSize: -1, + }), + /maxExportBatchSize must be a positive integer/ + ); + + assert.throws( + () => + new PeriodicExportingMetricReader({ + exporter: exporter, + exportIntervalMillis: 1, + exportTimeoutMillis: 1, + maxExportBatchSize: 1.5, + }), + /maxExportBatchSize must be a positive integer/ + ); + }); + it('should throw when timeout less or equal to interval', () => { const exporter = new TestDeltaMetricExporter(); assert.throws( @@ -479,6 +524,39 @@ describe('PeriodicExportingMetricReader', () => { exporter.throwExport = false; await reader.shutdown(); }); + + it('should not initiate collect when an export is ongoing', async () => { + const exporter = new TestMetricExporter(); + exporter.exportTime = 100; // Make it slow + const reader = new PeriodicExportingMetricReader({ + exporter: exporter, + exportIntervalMillis: MAX_32_BIT_INT, + exportTimeoutMillis: 80, + }); + + reader.setMetricProducer( + new TestMetricProducer({ resourceMetrics: resourceMetrics, errors: [] }) + ); + + const collectSpy = sinon.spy(reader, 'collect'); + + // Trigger first export + const firstFlush = reader.forceFlush(); + + // Wait a bit to ensure the first call has passed the `this.collect` call + // and is now in the export phase. + await new Promise(resolve => setTimeout(resolve, 10)); + + // Trigger second export + await reader.forceFlush(); + + // The second forceFlush should have skipped collection. + assert.strictEqual(collectSpy.callCount, 1); + + // Wait for the first export to complete to clean up + await firstFlush; + await reader.shutdown(); + }); }); describe('forceFlush', () => { @@ -782,5 +860,562 @@ describe('PeriodicExportingMetricReader', () => { await assert.rejects(() => reader.shutdown(), /Error during forceFlush/); }); + + describe('maxExportBatchSize', () => { + it('should split batches when exceeding maxExportBatchSize', async () => { + const exporter = new TestMetricExporter(); + const reader = new PeriodicExportingMetricReader({ + exporter: exporter, + exportIntervalMillis: MAX_32_BIT_INT, + maxExportBatchSize: 2, + }); + + const resourceMetrics: ResourceMetrics = { + resource: { + attributes: {}, + merge: sinon.stub(), + getRawAttributes: () => [], + } as any, + scopeMetrics: [ + { + scope: { name: 'test' }, + metrics: [ + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 1, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 2, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 3, + }, + ], + descriptor: { + name: 'm1', + description: '', + unit: '', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + ], + }; + + reader.setMetricProducer( + new TestMetricProducer({ + resourceMetrics: resourceMetrics, + errors: [], + }) + ); + + await reader.forceFlush(); + + const exports = exporter.getExports(); + assert.strictEqual(exports.length, 2); + + // First batch should have 2 data points + assert.strictEqual( + exports[0].scopeMetrics[0].metrics[0].dataPoints.length, + 2 + ); + assert.strictEqual( + exports[0].scopeMetrics[0].metrics[0].dataPoints[0].value, + 1 + ); + assert.strictEqual( + exports[0].scopeMetrics[0].metrics[0].dataPoints[1].value, + 2 + ); + + // Second batch should have 1 data point + assert.strictEqual( + exports[1].scopeMetrics[0].metrics[0].dataPoints.length, + 1 + ); + assert.strictEqual( + exports[1].scopeMetrics[0].metrics[0].dataPoints[0].value, + 3 + ); + + await reader.shutdown(); + }); + + it('should not export batches simultaneously when one times out', async () => { + const exporter = new TestMetricExporter(); + exporter.exportTime = 50; // Make export take some time + const reader = new PeriodicExportingMetricReader({ + exporter: exporter, + exportIntervalMillis: MAX_32_BIT_INT, + exportTimeoutMillis: 20, // Timeout smaller than export time + maxExportBatchSize: 1, + }); + + const resourceMetrics: ResourceMetrics = { + resource: { + attributes: {}, + merge: sinon.stub(), + getRawAttributes: () => [], + } as any, + scopeMetrics: [ + { + scope: { name: 'test' }, + metrics: [ + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 1, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 2, + }, + ], + descriptor: { + name: 'm1', + description: '', + unit: '', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + ], + }; + + const producer = new TestMetricProducer({ + resourceMetrics: resourceMetrics, + errors: [], + }); + reader.setMetricProducer(producer); + + await reader.forceFlush(); + + // Assert that they didn't overlap + assert.strictEqual( + exporter.maxConcurrentCalls, + 1, + 'Batches should not be exported simultaneously' + ); + + await reader.shutdown(); + }); + + it('should skip subsequent export when one is ongoing', async () => { + const exporter = new TestMetricExporter(); + exporter.exportTime = 50; // Make export take some time + const reader = new PeriodicExportingMetricReader({ + exporter: exporter, + exportIntervalMillis: MAX_32_BIT_INT, + }); + + const resourceMetrics: ResourceMetrics = { + resource: { + attributes: {}, + merge: sinon.stub(), + getRawAttributes: () => [], + } as any, + scopeMetrics: [ + { + scope: { name: 'test' }, + metrics: [ + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 1, + }, + ], + descriptor: { + name: 'm1', + description: '', + unit: '', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + ], + }; + + const producer = new TestMetricProducer({ + resourceMetrics: resourceMetrics, + errors: [], + }); + reader.setMetricProducer(producer); + + // Trigger first export + const p1 = reader.forceFlush(); + // Wait a bit to ensure the first call has passed the `this.collect` call + // and is now in the export phase. + await new Promise(resolve => setTimeout(resolve, 10)); + // Trigger second export + const p2 = reader.forceFlush(); + + await Promise.all([p1, p2]); + + const exports = exporter.getExports(); + assert.strictEqual(exports.length, 1); + + // Assert that they didn't overlap (only 1 ran) + assert.strictEqual(exporter.maxConcurrentCalls, 1); + + await reader.shutdown(); + }); + + it('should split data points across metrics if needed', async () => { + const exporter = new TestMetricExporter(); + const reader = new PeriodicExportingMetricReader({ + exporter: exporter, + exportIntervalMillis: MAX_32_BIT_INT, + maxExportBatchSize: 2, + }); + + const resourceMetrics: ResourceMetrics = { + resource: { + attributes: {}, + merge: sinon.stub(), + getRawAttributes: () => [], + } as any, + scopeMetrics: [ + { + scope: { name: 'test' }, + metrics: [ + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 1, + }, + ], + descriptor: { + name: 'm1', + description: '', + unit: '', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 2, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 3, + }, + ], + descriptor: { + name: 'm2', + description: '', + unit: '', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + ], + }; + + reader.setMetricProducer( + new TestMetricProducer({ + resourceMetrics: resourceMetrics, + errors: [], + }) + ); + + await reader.forceFlush(); + + const exports = exporter.getExports(); + assert.strictEqual(exports.length, 2); + + // First batch should have 2 data points (m1:1, m2:2) + assert.strictEqual(exports[0].scopeMetrics[0].metrics.length, 2); + assert.strictEqual( + exports[0].scopeMetrics[0].metrics[0].dataPoints.length, + 1 + ); + assert.strictEqual( + exports[0].scopeMetrics[0].metrics[0].dataPoints[0].value, + 1 + ); + assert.strictEqual( + exports[0].scopeMetrics[0].metrics[1].dataPoints.length, + 1 + ); + assert.strictEqual( + exports[0].scopeMetrics[0].metrics[1].dataPoints[0].value, + 2 + ); + + // Second batch should have 1 data point (m2:3) + assert.strictEqual(exports[1].scopeMetrics[0].metrics.length, 1); + assert.strictEqual( + exports[1].scopeMetrics[0].metrics[0].descriptor.name, + 'm2' + ); + assert.strictEqual( + exports[1].scopeMetrics[0].metrics[0].dataPoints.length, + 1 + ); + assert.strictEqual( + exports[1].scopeMetrics[0].metrics[0].dataPoints[0].value, + 3 + ); + + await reader.shutdown(); + }); + + it('should continue exporting remaining batches if one fails', async () => { + const exporter = new TestMetricExporter(); + exporter.rejectExport = true; // Fail all exports + const reader = new PeriodicExportingMetricReader({ + exporter: exporter, + exportIntervalMillis: MAX_32_BIT_INT, + maxExportBatchSize: 1, + }); + + const resourceMetrics: ResourceMetrics = { + resource: { + attributes: {}, + merge: sinon.stub(), + getRawAttributes: () => [], + } as any, + scopeMetrics: [ + { + scope: { name: 'test' }, + metrics: [ + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 1, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 2, + }, + ], + descriptor: { + name: 'm1', + description: '', + unit: '', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + ], + }; + + reader.setMetricProducer( + new TestMetricProducer({ + resourceMetrics: resourceMetrics, + errors: [], + }) + ); + + // Call forceFlush to trigger the export + await reader.forceFlush(); + + const exports = exporter.getExports(); + assert.strictEqual(exports.length, 2); // Both batches should have been attempted + + await reader.shutdown(); + }); + + it('should apply timeout to individual batches and not combination', async () => { + const clock = sinon.useFakeTimers(); + const exporter = new TestMetricExporter(); + exporter.exportTime = 15; // Each batch takes 15ms + + const reader = new PeriodicExportingMetricReader({ + exporter: exporter, + exportIntervalMillis: MAX_32_BIT_INT, + maxExportBatchSize: 1, // Results in 2 batches + exportTimeoutMillis: 20, // Timeout is 20ms + }); + + const resourceMetrics: ResourceMetrics = { + resource: { + attributes: {}, + merge: sinon.stub(), + getRawAttributes: () => [], + } as any, + scopeMetrics: [ + { + scope: { name: 'test' }, + metrics: [ + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 1, + }, + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 2, + }, + ], + descriptor: { + name: 'm1', + description: '', + unit: '', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + ], + }; + + reader.setMetricProducer( + new TestMetricProducer({ + resourceMetrics: resourceMetrics, + errors: [], + }) + ); + + // Call _runOnce directly to avoid forceFlush timer issues in this specific test + const runOncePromise = (reader as any)._runOnce(); + + // Batch 1 should be scheduled. Tick 15ms to complete it. + await clock.tickAsync(15); + + // Batch 2 should be scheduled. Tick 15ms to complete it. + await clock.tickAsync(15); + + await runOncePromise; + + const exports = exporter.getExports(); + assert.strictEqual(exports.length, 2); + + clock.restore(); + }); + + it('should log timeout error to diag and not propagate to globalErrorHandler', async () => { + const clock = sinon.useFakeTimers(); + const exporter = new TestMetricExporter(); + exporter.exportTime = 50; // Make it take 50ms + + const reader = new PeriodicExportingMetricReader({ + exporter: exporter, + exportIntervalMillis: MAX_32_BIT_INT, + maxExportBatchSize: 1, + exportTimeoutMillis: 20, // Timeout is 20ms + }); + + const resourceMetrics: ResourceMetrics = { + resource: { + attributes: {}, + merge: sinon.stub(), + getRawAttributes: () => [], + } as any, + scopeMetrics: [ + { + scope: { name: 'test' }, + metrics: [ + { + dataPointType: DataPointType.GAUGE, + dataPoints: [ + { + startTime: [0, 0], + endTime: [0, 0], + attributes: {}, + value: 1, + }, + ], + descriptor: { + name: 'm1', + description: '', + unit: '', + valueType: ValueType.INT, + }, + aggregationTemporality: AggregationTemporality.CUMULATIVE, + }, + ], + }, + ], + }; + + reader.setMetricProducer( + new TestMetricProducer({ + resourceMetrics: resourceMetrics, + errors: [], + }) + ); + + const diagErrorStub = sinon.stub(diag, 'error'); + const errorHandlerStub = sinon.stub(); + setGlobalErrorHandler(errorHandlerStub); + + const runOncePromise = (reader as any)._runOnce(); + + // Tick 20ms to trigger timeout + await clock.tickAsync(20); + + await runOncePromise; + + sinon.assert.calledOnce(diagErrorStub); + assert.match( + diagErrorStub.firstCall.args[0], + /metrics export timed out/ + ); + + // Global error handler should not be called for timeout errors + sinon.assert.notCalled(errorHandlerStub); + + // Restore global error handler + setGlobalErrorHandler(() => {}); + clock.restore(); + }); + }); }); });