diff --git a/metrics_api/Gemfile b/metrics_api/Gemfile index 1868cf7f2..a1e53ddc5 100644 --- a/metrics_api/Gemfile +++ b/metrics_api/Gemfile @@ -12,6 +12,7 @@ eval_gemfile '../contrib/Gemfile.shared' group :test, :development do gem 'opentelemetry-api', path: '../api', require: false + gem 'opentelemetry-metrics-sdk', path: '../metrics_sdk', require: false gem 'pry' gem 'pry-byebug' unless RUBY_ENGINE == 'jruby' end diff --git a/metrics_api/Rakefile b/metrics_api/Rakefile index d202bf4c4..a6ac35404 100644 --- a/metrics_api/Rakefile +++ b/metrics_api/Rakefile @@ -29,3 +29,18 @@ default_tasks = end task default: default_tasks + +namespace :bench do + bench_files = FileList['benchmarks/*_bench.rb'] + + bench_files.each do |file| + name = File.basename(file, '_bench.rb') + desc "Run #{name} benchmarks" + task name do + sh "bundle exec ruby #{file}" + end + end + + desc 'Run all benchmarks' + task all: bench_files.map { |f| File.basename(f, '_bench.rb') } +end diff --git a/metrics_api/benchmarks/README.md b/metrics_api/benchmarks/README.md new file mode 100644 index 000000000..f07b8e3b6 --- /dev/null +++ b/metrics_api/benchmarks/README.md @@ -0,0 +1,137 @@ +# Metrics API Benchmarks + +This directory contains [benchmark-ips](https://github.com/evanphx/benchmark-ips) benchmarks for the OpenTelemetry Ruby Metrics API and SDK. They cover no-op API recording, SDK instrument recording, attribute cardinality, views, exemplar filters, exemplar reservoirs, and aggregations. + +## Running the Benchmarks + +Run from the `metrics_api/` directory, adding each sibling gem's `lib/` to the load path: + +```bash +bundle exec ruby benchmarks/aggregation_bench.rb +bundle exec ruby benchmarks/attributes_bench.rb +bundle exec ruby benchmarks/exemplar_filter_bench.rb +bundle exec ruby benchmarks/exemplar_reservoir_bench.rb +bundle exec ruby benchmarks/instrument_bench.rb +bundle exec ruby benchmarks/noop_instrument_bench.rb +bundle exec ruby benchmarks/view_bench.rb +``` + +## Benchmark Files + +| File | What it measures | +| ---- | --------------- | +| `instrument_bench.rb` | Real SDK instruments — all synchronous types (counter, histogram, gauge, up-down counter) | +| `noop_instrument_bench.rb` | No-op API instruments only — micro-benchmark of the lightest possible instrumentation layer | +| `attributes_bench.rb` | How attribute set size (0 / 1 / 3 / 8 keys) affects SDK counter throughput | +| `view_bench.rb` | Impact of zero, one matching, one non-matching, and three matching registered views | +| `exemplar_filter_bench.rb` | Exemplar filter cost (`AlwaysOff` / `AlwaysOn` / `TraceBased`) on SDK counter | +| `exemplar_reservoir_bench.rb` | Exemplar reservoir cost (`Noop` / `SimpleFixedSize` / `AlignedHistogramBucket`) with `AlwaysOn` filter | +| `aggregation_bench.rb` | Histogram recording throughput across all five aggregations (`Drop`, `Sum`, `LastValue`, `ExplicitBucketHistogram`, `ExponentialBucketHistogram`) | + +## Sample Run + +### System Specifications + +**OS Information:** + +- Distribution: Ubuntu 24.04.3 LTS (Noble Numbat) +- Kernel: Linux 6.14.0-1018-aws +- Architecture: x86_64 + +**Memory:** + +- Total: ~3.91 GB (4,006,000 kB) +- Available: ~3.40 GB (3,470,496 kB) +- Free: ~3.13 GB (3,195,972 kB) + +**CPU:** + +- Processor: Intel(R) Xeon(R) CPU E5-2686 v4 @ 2.30GHz +- Cores: 2 +- Threads: 2 +- Virtualization: Xen (Full) +- Cache: L1d 64 KiB × 2 | L1i 64 KiB × 2 | L2 512 KiB × 2 | L3 45 MiB + +**Runtime:** + +- Ruby: 3.4.0dev (2024-12-25 master f450108330) +PRISM [x86_64-linux] + +### Benchmark Results + +#### Aggregation Benchmarks + +`bundle exec ruby benchmarks/aggregation_bench.rb` + +| Aggregation Type | Throughput | Time/Op | Relative Performance | +| --- | --- | --- | --- | +| Sum | 172,228.5 i/s | 5.81 μs/i | **Fastest** | +| Drop | 163,185.4 i/s | 6.13 μs/i | 1.06x slower | +| LastValue | 155,601.6 i/s | 6.43 μs/i | 1.11x slower | +| ExplicitBucketHistogram | 153,229.4 i/s | 6.53 μs/i | 1.12x slower | +| ExponentialBucketHistogram | 83,628.0 i/s | 11.96 μs/i | 2.06x slower | + +#### Attribute Cardinality (SDK Counter) + +`bundle exec ruby benchmarks/attributes_bench.rb` + +| Attribute Count | Throughput | Time/Op | Relative Performance | +| --- | --- | --- | --- | +| No attrs (0) | 235,949.5 i/s | 4.24 μs/i | **Fastest** | +| Small attrs (1) | 223,849.7 i/s | 4.47 μs/i | 1.05x slower | +| Medium attrs (3) | 216,702.4 i/s | 4.61 μs/i | 1.09x slower | +| Large attrs (8) | 197,824.3 i/s | 5.05 μs/i | 1.19x slower | + +#### Exemplar Filters + +`bundle exec ruby benchmarks/exemplar_filter_bench.rb` + +| Filter Type | Throughput | Time/Op | Relative Performance | +| --- | --- | --- | --- | +| AlwaysOff | 227,492.0 i/s | 4.40 μs/i | **Fastest** | +| TraceBased | 211,242.6 i/s | 4.73 μs/i | 1.08x slower | +| AlwaysOn | 133,263.6 i/s | 7.50 μs/i | 1.71x slower | + +#### Exemplar Reservoirs + +`bundle exec ruby benchmarks/exemplar_reservoir_bench.rb` + +| Instrument | Reservoir Type | Throughput | Time/Op | Relative Performance | +| --- | --- | --- | --- | --- | +| Counter | SimpleFixedSize | 126,669.7 i/s | 7.89 μs/i | **Fastest** | +| Histogram | SimpleFixedSize | 118,602.8 i/s | 8.43 μs/i | 1.07x slower | +| Histogram | AlignedHistogramBucket | 113,535.5 i/s | 8.81 μs/i | 1.12x slower | +| Counter | Noop | 25,391.0 i/s | 39.38 μs/i | 4.99x slower | +| Histogram | Noop | 25,329.6 i/s | 39.48 μs/i | 5.00x slower | + +#### Synchronous Instruments (SDK) + +`bundle exec ruby benchmarks/instrument_bench.rb` + +| Instrument | Throughput | Time/Op | Relative Performance | +| --- | --- | --- | --- | +| up_down_counter#add | 225,684.1 i/s | 4.43 μs/i | **Fastest** | +| counter#add | 222,590.0 i/s | 4.49 μs/i | ~Same | +| gauge#record | 202,016.1 i/s | 4.95 μs/i | 1.12x slower | +| histogram#record | 199,490.0 i/s | 5.01 μs/i | 1.13x slower | + +#### No-Op Instruments (API) + +`bundle exec ruby benchmarks/noop_instrument_bench.rb` + +| Instrument | Throughput | Time/Op | Relative Performance | +| --- | --- | --- | --- | +| noop up_down_counter#add | 4,342,256.4 i/s | 230.30 ns/i | **Fastest** | +| noop counter#add | 4,334,817.2 i/s | 230.69 ns/i | ~Same | +| noop gauge#record | 4,331,789.5 i/s | 230.85 ns/i | ~Same | +| noop histogram#record | 4,329,196.6 i/s | 230.99 ns/i | ~Same | + +#### View Impact on Performance + +`bundle exec ruby benchmarks/view_bench.rb` + +| Views | Throughput | Time/Op | Relative Performance | +| --- | --- | --- | --- | +| No views | 225,245.4 i/s | 4.44 μs/i | **Fastest** | +| 1 non-matching | 224,709.9 i/s | 4.45 μs/i | ~Same | +| 1 matching | 169,952.5 i/s | 5.88 μs/i | 1.33x slower | +| 3 matching | 62,762.3 i/s | 15.93 μs/i | 3.59x slower | diff --git a/metrics_api/benchmarks/aggregation_bench.rb b/metrics_api/benchmarks/aggregation_bench.rb new file mode 100644 index 000000000..4614772bd --- /dev/null +++ b/metrics_api/benchmarks/aggregation_bench.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require_relative 'bench_helper' + +def histogram_with_agg(aggregation) + provider = OpenTelemetry::SDK::Metrics::MeterProvider.new + provider.add_view('bench.agg.histogram', aggregation: aggregation) + provider.add_metric_reader(new_reader) + provider.meter('bench').create_histogram('bench.agg.histogram') +end + +explicit_hist = histogram_with_agg(Agg::ExplicitBucketHistogram.new) +exponential_hist = histogram_with_agg(Agg::ExponentialBucketHistogram.new) +sum_hist = histogram_with_agg(Agg::Sum.new) +last_value_hist = histogram_with_agg(Agg::LastValue.new) +drop_hist = histogram_with_agg(Agg::Drop.new) + +Benchmark.ips do |x| + x.report('histogram ExplicitBucketHistogram') { explicit_hist.record(42) } + x.report('histogram ExponentialBucketHistogram') { exponential_hist.record(42) } + x.report('histogram Sum aggregation') { sum_hist.record(42) } + x.report('histogram LastValue aggregation') { last_value_hist.record(42) } + x.report('histogram Drop aggregation') { drop_hist.record(42) } + x.compare! +end diff --git a/metrics_api/benchmarks/attributes_bench.rb b/metrics_api/benchmarks/attributes_bench.rb new file mode 100644 index 000000000..bac355df2 --- /dev/null +++ b/metrics_api/benchmarks/attributes_bench.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require_relative 'bench_helper' + +ATTRS_NONE = {}.freeze +ATTRS_SMALL = { 'env' => 'prod' }.freeze +ATTRS_MEDIUM = { 'http.method' => 'GET', 'http.status_code' => 200, 'http.route' => '/api/users' }.freeze +ATTRS_LARGE = { + 'http.method' => 'GET', + 'http.status_code' => 200, + 'http.route' => '/api/users', + 'net.host.name' => 'example.com', + 'net.host.port' => 443, + 'http.scheme' => 'https', + 'http.flavor' => '1.1', + 'http.user_agent' => 'Ruby/3.3' +}.freeze + +puts "\n#{'=' * 60}" +puts '= Attribute cardinality (SDK counter)' +puts '=' * 60 + +card_counter = build_sdk_meter.create_counter('bench.cardinality.counter') + +Benchmark.ips do |x| + x.report('counter#add (no attrs)') { card_counter.add(1, attributes: ATTRS_NONE) } + x.report('counter#add (small attrs)') { card_counter.add(1, attributes: ATTRS_SMALL) } + x.report('counter#add (medium attrs)') { card_counter.add(1, attributes: ATTRS_MEDIUM) } + x.report('counter#add (large attrs)') { card_counter.add(1, attributes: ATTRS_LARGE) } + x.compare! +end diff --git a/metrics_api/benchmarks/bench_helper.rb b/metrics_api/benchmarks/bench_helper.rb new file mode 100644 index 000000000..3aa407926 --- /dev/null +++ b/metrics_api/benchmarks/bench_helper.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'benchmark/ips' +require 'opentelemetry-metrics-api' +require 'opentelemetry/sdk/metrics' + +OpenTelemetry.logger = Logger.new(File::NULL) + +Agg = OpenTelemetry::SDK::Metrics::Aggregation +Ex = OpenTelemetry::SDK::Metrics::Exemplar +Export = OpenTelemetry::SDK::Metrics::Export + +def new_reader + Export::InMemoryMetricPullExporter.new +end + +def counter_with(exemplar_filter:, exemplar_reservoir: nil) + meter = build_sdk_meter(exemplar_filter: exemplar_filter) + meter.create_counter('bench.exemplar.counter', exemplar_reservoir: exemplar_reservoir) +end + +def histogram_with(exemplar_filter:, exemplar_reservoir: nil) + meter = build_sdk_meter(exemplar_filter: exemplar_filter) + meter.create_histogram('bench.exemplar.histogram', exemplar_reservoir: exemplar_reservoir) +end + +def build_sdk_meter(exemplar_filter: nil, views: []) + provider = OpenTelemetry::SDK::Metrics::MeterProvider.new + provider.enable_exemplar_filter(exemplar_filter: exemplar_filter) if exemplar_filter + views.each { |(name, opts)| provider.add_view(name, **opts) } + provider.add_metric_reader(new_reader) + provider.meter('bench') +end diff --git a/metrics_api/benchmarks/exemplar_filter_bench.rb b/metrics_api/benchmarks/exemplar_filter_bench.rb new file mode 100644 index 000000000..8dd516a3c --- /dev/null +++ b/metrics_api/benchmarks/exemplar_filter_bench.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require_relative 'bench_helper' + +always_off_counter = counter_with(exemplar_filter: Ex::AlwaysOffExemplarFilter) +always_on_counter = counter_with(exemplar_filter: Ex::AlwaysOnExemplarFilter) +trace_based_counter = counter_with(exemplar_filter: Ex::TraceBasedExemplarFilter) + +Benchmark.ips do |x| + x.report('counter#add AlwaysOff filter') { always_off_counter.add(1) } + x.report('counter#add AlwaysOn filter') { always_on_counter.add(1) } + x.report('counter#add TraceBased filter') { trace_based_counter.add(1) } + x.compare! +end diff --git a/metrics_api/benchmarks/exemplar_reservoir_bench.rb b/metrics_api/benchmarks/exemplar_reservoir_bench.rb new file mode 100644 index 000000000..eb7d1360f --- /dev/null +++ b/metrics_api/benchmarks/exemplar_reservoir_bench.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require_relative 'bench_helper' + +noop_res_counter = counter_with(exemplar_filter: Ex::AlwaysOnExemplarFilter, + exemplar_reservoir: Ex::NoopExemplarReservoir.new) +simple_res_counter = counter_with(exemplar_filter: Ex::AlwaysOnExemplarFilter, + exemplar_reservoir: Ex::SimpleFixedSizeExemplarReservoir.new) + +aligned_histogram = histogram_with(exemplar_filter: Ex::AlwaysOnExemplarFilter, + exemplar_reservoir: Ex::AlignedHistogramBucketExemplarReservoir.new) +simple_histogram = histogram_with(exemplar_filter: Ex::AlwaysOnExemplarFilter, + exemplar_reservoir: Ex::SimpleFixedSizeExemplarReservoir.new) +noop_histogram = histogram_with(exemplar_filter: Ex::AlwaysOnExemplarFilter, + exemplar_reservoir: Ex::NoopExemplarReservoir.new) + +Benchmark.ips do |x| + x.report('counter Noop reservoir') { noop_res_counter.add(1) } + x.report('counter SimpleFixedSize reservoir') { simple_res_counter.add(1) } + x.report('histogram AlignedHistogramBucket') { aligned_histogram.record(42) } + x.report('histogram SimpleFixedSize reservoir') { simple_histogram.record(42) } + x.report('histogram Noop reservoir') { noop_histogram.record(42) } + x.compare! +end diff --git a/metrics_api/benchmarks/instrument_bench.rb b/metrics_api/benchmarks/instrument_bench.rb new file mode 100644 index 000000000..ac25a5fdf --- /dev/null +++ b/metrics_api/benchmarks/instrument_bench.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require_relative 'bench_helper' + +sdk_meter = build_sdk_meter +sdk_counter = sdk_meter.create_counter('bench.sdk.counter') +sdk_histogram = sdk_meter.create_histogram('bench.sdk.histogram') +sdk_gauge = sdk_meter.create_gauge('bench.sdk.gauge') +sdk_updown = sdk_meter.create_up_down_counter('bench.sdk.updown') + +Benchmark.ips do |x| + x.report('SDK counter#add') { sdk_counter.add(1) } + x.report('SDK histogram#record') { sdk_histogram.record(1) } + x.report('SDK gauge#record') { sdk_gauge.record(1) } + x.report('SDK up_down_counter#add') { sdk_updown.add(1) } + x.compare! +end diff --git a/metrics_api/benchmarks/noop_instrument_bench.rb b/metrics_api/benchmarks/noop_instrument_bench.rb new file mode 100644 index 000000000..e574f2fca --- /dev/null +++ b/metrics_api/benchmarks/noop_instrument_bench.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require_relative 'bench_helper' + +noop_meter = OpenTelemetry::Metrics::Meter.new +noop_counter = noop_meter.create_counter('bench.noop.counter') +noop_histogram = noop_meter.create_histogram('bench.noop.histogram') +noop_gauge = noop_meter.create_gauge('bench.noop.gauge') +noop_updown = noop_meter.create_up_down_counter('bench.noop.updown') + +Benchmark.ips do |x| + x.report('noop counter#add') { noop_counter.add(1) } + x.report('noop histogram#record') { noop_histogram.record(1) } + x.report('noop gauge#record') { noop_gauge.record(1) } + x.report('noop up_down_counter#add') { noop_updown.add(1) } + x.compare! +end diff --git a/metrics_api/benchmarks/view_bench.rb b/metrics_api/benchmarks/view_bench.rb new file mode 100644 index 000000000..e454438e8 --- /dev/null +++ b/metrics_api/benchmarks/view_bench.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require_relative 'bench_helper' + +# No view registered +no_view_counter = build_sdk_meter.create_counter('bench.view.counter') + +# One matching view +match_counter = build_sdk_meter( + views: [['bench.view.counter', { aggregation: Agg::Sum.new }]] +).create_counter('bench.view.counter') + +# One non-matching view (different instrument name) +nomatch_counter = build_sdk_meter( + views: [['other.counter', { aggregation: Agg::Sum.new }]] +).create_counter('bench.view.counter') + +# Three matching views +multi_provider = OpenTelemetry::SDK::Metrics::MeterProvider.new +3.times { multi_provider.add_view('bench.view.counter', aggregation: Agg::Sum.new) } +multi_provider.add_metric_reader(new_reader) +multi_counter = multi_provider.meter('bench').create_counter('bench.view.counter') + +Benchmark.ips do |x| + x.report('counter#add (no view registered)') { no_view_counter.add(1) } + x.report('counter#add (1 non-matching view)') { nomatch_counter.add(1) } + x.report('counter#add (1 matching view)') { match_counter.add(1) } + x.report('counter#add (3 matching views)') { multi_counter.add(1) } + x.compare! +end diff --git a/metrics_api/opentelemetry-metrics-api.gemspec b/metrics_api/opentelemetry-metrics-api.gemspec index 04766ca4a..71bb44cc0 100644 --- a/metrics_api/opentelemetry-metrics-api.gemspec +++ b/metrics_api/opentelemetry-metrics-api.gemspec @@ -26,6 +26,7 @@ Gem::Specification.new do |spec| spec.add_dependency 'opentelemetry-api', '~> 1.0' + spec.add_development_dependency 'benchmark-ips', '~> 2.14.0' spec.add_development_dependency 'minitest', '~> 5.0' spec.add_development_dependency 'opentelemetry-test-helpers', '~> 0.3.0' spec.add_development_dependency 'rake', '~> 13.3'