From fdbcd5e975ee206ac5eb55347a46bdc256cd6d73 Mon Sep 17 00:00:00 2001 From: Donal O'Sullivan Date: Fri, 24 Apr 2026 15:56:10 +0100 Subject: [PATCH 01/15] Add profiling to the otel demo and use pyroscope as the backend OTel collector added for profiling with highenend privileges. Send the profiling data to a pyroscope backend. Makefile updated with start-profiling target which starts the demo along with the profiling collector. --- Makefile | 4 +++ docker-compose-profiling.yml | 30 +++++++++++++++++++ .../provisioning/datasources/pyroscope.yaml | 8 +++++ src/otel-collector/otelcol-ebpf-profiling.yml | 18 +++++++++++ src/pyroscope/config.yaml | 9 ++++++ 5 files changed, 69 insertions(+) create mode 100644 docker-compose-profiling.yml create mode 100644 src/grafana/provisioning/datasources/pyroscope.yaml create mode 100644 src/otel-collector/otelcol-ebpf-profiling.yml create mode 100644 src/pyroscope/config.yaml diff --git a/Makefile b/Makefile index 3bd80f9140..4e8caa391a 100644 --- a/Makefile +++ b/Makefile @@ -254,3 +254,7 @@ endif .PHONY: build-react-native-android build-react-native-android: docker build -f src/react-native-app/android.Dockerfile --platform=linux/amd64 --output=. src/react-native-app + +.PHONY: start-profiling +start-profiling: + $(DOCKER_COMPOSE_CMD) $(DOCKER_COMPOSE_ENV) -f docker-compose.yml -f docker-compose-profiling.yml up --force-recreate --remove-orphans --detach diff --git a/docker-compose-profiling.yml b/docker-compose-profiling.yml new file mode 100644 index 0000000000..5d905f796d --- /dev/null +++ b/docker-compose-profiling.yml @@ -0,0 +1,30 @@ +services: + otel-ebpf-profiler: + image: otel/opentelemetry-collector-ebpf-profiler:0.147.0 + container_name: otel-ebpf-profiler + command: + - --config=/etc/ebpf-profiler-config.yaml + - --feature-gates=service.profilesSupport + privileged: true + pid: "host" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./src/otel-collector/otelcol-ebpf-profiling.yml:/etc/ebpf-profiler-config.yaml:ro + - /sys/kernel/debug:/sys/kernel/debug + - /sys/kernel/tracing:/sys/kernel/tracing + - /sys/fs/cgroup:/sys/fs/cgroup:ro + - /proc:/proc:ro + depends_on: + - pyroscope + pyroscope: + image: grafana/pyroscope:1.18.1 + container_name: pyroscope + command: + - -config.file=/etc/pyroscope.yaml + volumes: + - ./src/pyroscope/config.yaml:/etc/pyroscope.yaml:ro + ports: + - "${PYROSCOPE_PORT:-4040}:4040" + grafana: + environment: + - "GF_INSTALL_PLUGINS=grafana-opensearch-datasource,grafana-pyroscope-app" diff --git a/src/grafana/provisioning/datasources/pyroscope.yaml b/src/grafana/provisioning/datasources/pyroscope.yaml new file mode 100644 index 0000000000..234172ac5b --- /dev/null +++ b/src/grafana/provisioning/datasources/pyroscope.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 +datasources: + - name: Pyroscope + type: grafana-pyroscope-datasource + uid: pyroscope + url: http://pyroscope:4040 + isDefault: false + editable: true diff --git a/src/otel-collector/otelcol-ebpf-profiling.yml b/src/otel-collector/otelcol-ebpf-profiling.yml new file mode 100644 index 0000000000..315fb88c67 --- /dev/null +++ b/src/otel-collector/otelcol-ebpf-profiling.yml @@ -0,0 +1,18 @@ +receivers: + profiling: + +processors: + resourcedetection: + detectors: [docker] + +exporters: + otlphttp/pyroscope: + endpoint: http://pyroscope:4040 + +service: + pipelines: + profiles: + receivers: [profiling] + processors: [resourcedetection] + exporters: [otlphttp/pyroscope] + diff --git a/src/pyroscope/config.yaml b/src/pyroscope/config.yaml new file mode 100644 index 0000000000..93e32e408b --- /dev/null +++ b/src/pyroscope/config.yaml @@ -0,0 +1,9 @@ +storage: + backend: filesystem + filesystem: + dir: /data +limits: + ingestion_relabeling_rules: + - action: labelmap + regex: ^process\.executable\.name$ + replacement: service_name From 2b4fb6c689ed90f9a31d2028069599b9ae1645ac Mon Sep 17 00:00:00 2001 From: Donal O'Sullivan Date: Fri, 24 Apr 2026 16:16:56 +0100 Subject: [PATCH 02/15] Add profiling changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed9fcf62f0..8c214c3383 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ the release. ## Unreleased +* [profiling] Add profiling and use Pyroscope as the backend to ingest profiles. + This allows us to view profiles in Grafana. + [#3312](https://github.com/open-telemetry/opentelemetry-demo/pull/3312) * [docker] Podman doesn't support the tag feature of docker logs, for the otel-demo to support podman we need to remove the tag from docker logs. [#3304](https://github.com/open-telemetry/opentelemetry-demo/pull/3304) From b10552547d34d170dcd86bc706eee1733afd9470 Mon Sep 17 00:00:00 2001 From: Donal O'Sullivan Date: Fri, 24 Apr 2026 16:21:16 +0100 Subject: [PATCH 03/15] Fix yamllint errros with profiling yaml files --- docker-compose-profiling.yml | 58 +++++++++---------- src/otel-collector/otelcol-ebpf-profiling.yml | 1 - src/pyroscope/config.yaml | 6 +- 3 files changed, 32 insertions(+), 33 deletions(-) diff --git a/docker-compose-profiling.yml b/docker-compose-profiling.yml index 5d905f796d..f295680788 100644 --- a/docker-compose-profiling.yml +++ b/docker-compose-profiling.yml @@ -1,30 +1,30 @@ services: - otel-ebpf-profiler: - image: otel/opentelemetry-collector-ebpf-profiler:0.147.0 - container_name: otel-ebpf-profiler - command: - - --config=/etc/ebpf-profiler-config.yaml - - --feature-gates=service.profilesSupport - privileged: true - pid: "host" - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - - ./src/otel-collector/otelcol-ebpf-profiling.yml:/etc/ebpf-profiler-config.yaml:ro - - /sys/kernel/debug:/sys/kernel/debug - - /sys/kernel/tracing:/sys/kernel/tracing - - /sys/fs/cgroup:/sys/fs/cgroup:ro - - /proc:/proc:ro - depends_on: - - pyroscope - pyroscope: - image: grafana/pyroscope:1.18.1 - container_name: pyroscope - command: - - -config.file=/etc/pyroscope.yaml - volumes: - - ./src/pyroscope/config.yaml:/etc/pyroscope.yaml:ro - ports: - - "${PYROSCOPE_PORT:-4040}:4040" - grafana: - environment: - - "GF_INSTALL_PLUGINS=grafana-opensearch-datasource,grafana-pyroscope-app" + otel-ebpf-profiler: + image: otel/opentelemetry-collector-ebpf-profiler:0.147.0 + container_name: otel-ebpf-profiler + command: + - --config=/etc/ebpf-profiler-config.yaml + - --feature-gates=service.profilesSupport + privileged: true + pid: "host" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./src/otel-collector/otelcol-ebpf-profiling.yml:/etc/ebpf-profiler-config.yaml:ro + - /sys/kernel/debug:/sys/kernel/debug + - /sys/kernel/tracing:/sys/kernel/tracing + - /sys/fs/cgroup:/sys/fs/cgroup:ro + - /proc:/proc:ro + depends_on: + - pyroscope + pyroscope: + image: grafana/pyroscope:1.18.1 + container_name: pyroscope + command: + - -config.file=/etc/pyroscope.yaml + volumes: + - ./src/pyroscope/config.yaml:/etc/pyroscope.yaml:ro + ports: + - "${PYROSCOPE_PORT:-4040}:4040" + grafana: + environment: + - "GF_INSTALL_PLUGINS=grafana-opensearch-datasource,grafana-pyroscope-app" diff --git a/src/otel-collector/otelcol-ebpf-profiling.yml b/src/otel-collector/otelcol-ebpf-profiling.yml index 315fb88c67..57d7996cf5 100644 --- a/src/otel-collector/otelcol-ebpf-profiling.yml +++ b/src/otel-collector/otelcol-ebpf-profiling.yml @@ -15,4 +15,3 @@ service: receivers: [profiling] processors: [resourcedetection] exporters: [otlphttp/pyroscope] - diff --git a/src/pyroscope/config.yaml b/src/pyroscope/config.yaml index 93e32e408b..6e6702bb5c 100644 --- a/src/pyroscope/config.yaml +++ b/src/pyroscope/config.yaml @@ -4,6 +4,6 @@ storage: dir: /data limits: ingestion_relabeling_rules: - - action: labelmap - regex: ^process\.executable\.name$ - replacement: service_name + - action: labelmap + regex: ^process\.executable\.name$ + replacement: service_name From 17edffc713090899aafaf2100048a5cbc779564c Mon Sep 17 00:00:00 2001 From: Donal O'Sullivan Date: Fri, 24 Apr 2026 16:32:30 +0100 Subject: [PATCH 04/15] Add OTel license to the top of the new profiling yaml files --- docker-compose-profiling.yml | 3 +++ src/grafana/provisioning/datasources/pyroscope.yaml | 3 +++ src/otel-collector/otelcol-ebpf-profiling.yml | 3 +++ src/pyroscope/config.yaml | 3 +++ 4 files changed, 12 insertions(+) diff --git a/docker-compose-profiling.yml b/docker-compose-profiling.yml index f295680788..09bffc0cce 100644 --- a/docker-compose-profiling.yml +++ b/docker-compose-profiling.yml @@ -1,3 +1,6 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + services: otel-ebpf-profiler: image: otel/opentelemetry-collector-ebpf-profiler:0.147.0 diff --git a/src/grafana/provisioning/datasources/pyroscope.yaml b/src/grafana/provisioning/datasources/pyroscope.yaml index 234172ac5b..3ce80ff367 100644 --- a/src/grafana/provisioning/datasources/pyroscope.yaml +++ b/src/grafana/provisioning/datasources/pyroscope.yaml @@ -1,3 +1,6 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + apiVersion: 1 datasources: - name: Pyroscope diff --git a/src/otel-collector/otelcol-ebpf-profiling.yml b/src/otel-collector/otelcol-ebpf-profiling.yml index 57d7996cf5..6f05fc51cd 100644 --- a/src/otel-collector/otelcol-ebpf-profiling.yml +++ b/src/otel-collector/otelcol-ebpf-profiling.yml @@ -1,3 +1,6 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + receivers: profiling: diff --git a/src/pyroscope/config.yaml b/src/pyroscope/config.yaml index 6e6702bb5c..56b3e32935 100644 --- a/src/pyroscope/config.yaml +++ b/src/pyroscope/config.yaml @@ -1,3 +1,6 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + storage: backend: filesystem filesystem: From c9df5d9f36e1b6f028d2dae98f51bec2958a603e Mon Sep 17 00:00:00 2001 From: Donal O'Sullivan Date: Thu, 30 Apr 2026 16:09:15 +0100 Subject: [PATCH 05/15] Remove grafana pyroscope and use firepit as the profiling backend Firepit is a opensource tool to view profiles in a flame graph, it doesn't store the profiles. https://github.com/florianl/firepit/tree/main --- docker-compose-profiling.yml | 19 +++++++------------ .../provisioning/datasources/pyroscope.yaml | 11 ----------- src/otel-collector/otelcol-ebpf-profiling.yml | 17 ++++++++++++++--- src/pyroscope/config.yaml | 12 ------------ 4 files changed, 21 insertions(+), 38 deletions(-) delete mode 100644 src/grafana/provisioning/datasources/pyroscope.yaml delete mode 100644 src/pyroscope/config.yaml diff --git a/docker-compose-profiling.yml b/docker-compose-profiling.yml index 09bffc0cce..c0921d5d0a 100644 --- a/docker-compose-profiling.yml +++ b/docker-compose-profiling.yml @@ -18,16 +18,11 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:ro - /proc:/proc:ro depends_on: - - pyroscope - pyroscope: - image: grafana/pyroscope:1.18.1 - container_name: pyroscope - command: - - -config.file=/etc/pyroscope.yaml - volumes: - - ./src/pyroscope/config.yaml:/etc/pyroscope.yaml:ro + - firepit + firepit: + image: ghcr.io/florianl/firepit:latest + build: + context: https://github.com/florianl/firepit.git + container_name: firepit ports: - - "${PYROSCOPE_PORT:-4040}:4040" - grafana: - environment: - - "GF_INSTALL_PLUGINS=grafana-opensearch-datasource,grafana-pyroscope-app" + - "${FIREPIT_PORT:-4040}:8080" diff --git a/src/grafana/provisioning/datasources/pyroscope.yaml b/src/grafana/provisioning/datasources/pyroscope.yaml deleted file mode 100644 index 3ce80ff367..0000000000 --- a/src/grafana/provisioning/datasources/pyroscope.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: 1 -datasources: - - name: Pyroscope - type: grafana-pyroscope-datasource - uid: pyroscope - url: http://pyroscope:4040 - isDefault: false - editable: true diff --git a/src/otel-collector/otelcol-ebpf-profiling.yml b/src/otel-collector/otelcol-ebpf-profiling.yml index 6f05fc51cd..d2c4a10017 100644 --- a/src/otel-collector/otelcol-ebpf-profiling.yml +++ b/src/otel-collector/otelcol-ebpf-profiling.yml @@ -3,18 +3,29 @@ receivers: profiling: + samples_per_second: 10 + reporter_interval: 10s + monitor_interval: 5s + off_cpu_threshold: 1.0 processors: resourcedetection: detectors: [docker] + resource: + attributes: + - key: service.name + from_attribute: container.name + action: upsert exporters: - otlphttp/pyroscope: - endpoint: http://pyroscope:4040 + otlp/firepit: + endpoint: firepit:4317 + tls: + insecure: true service: pipelines: profiles: receivers: [profiling] processors: [resourcedetection] - exporters: [otlphttp/pyroscope] + exporters: [otlp/firepit] diff --git a/src/pyroscope/config.yaml b/src/pyroscope/config.yaml deleted file mode 100644 index 56b3e32935..0000000000 --- a/src/pyroscope/config.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -storage: - backend: filesystem - filesystem: - dir: /data -limits: - ingestion_relabeling_rules: - - action: labelmap - regex: ^process\.executable\.name$ - replacement: service_name From 63fc06fe7e3f94077800a200a76cfe68c09ba91a Mon Sep 17 00:00:00 2001 From: Donal O'Sullivan Date: Thu, 30 Apr 2026 16:13:22 +0100 Subject: [PATCH 06/15] Update changelog for profiling to specify firepit --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c214c3383..562b4c07b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,8 @@ the release. ## Unreleased -* [profiling] Add profiling and use Pyroscope as the backend to ingest profiles. - This allows us to view profiles in Grafana. +* [profiling] Add profiling and use firepit as the backend to ingest profiles. + This allows us to view profiles in the firepit webui. [#3312](https://github.com/open-telemetry/opentelemetry-demo/pull/3312) * [docker] Podman doesn't support the tag feature of docker logs, for the otel-demo to support podman we need to remove the tag from docker logs. From 9def92f311142e47f476b01d25600b807300bf47 Mon Sep 17 00:00:00 2001 From: Donal O'Sullivan Date: Fri, 1 May 2026 10:07:15 +0100 Subject: [PATCH 07/15] Send the profiles from the ebpf collector to the main collector This shows how you can use the main collector as a gateway for all otel signals --- Makefile | 2 +- docker-compose-profiling.yml => compose-profiling.yml | 5 ++++- docker-compose.yml | 5 ++++- src/otel-collector/otelcol-config.yml | 8 ++++++++ src/otel-collector/otelcol-ebpf-profiling.yml | 9 +++------ 5 files changed, 20 insertions(+), 9 deletions(-) rename docker-compose-profiling.yml => compose-profiling.yml (89%) diff --git a/Makefile b/Makefile index ffe728e435..9b75a87d48 100644 --- a/Makefile +++ b/Makefile @@ -257,4 +257,4 @@ build-react-native-android: .PHONY: start-profiling start-profiling: - $(DOCKER_COMPOSE_CMD) $(DOCKER_COMPOSE_ENV) -f docker-compose.yml -f docker-compose-profiling.yml up --force-recreate --remove-orphans --detach + $(DOCKER_COMPOSE_CMD) $(DOCKER_COMPOSE_ENV) -f docker-compose.yml -f compose-profiling.yml up --force-recreate --remove-orphans --detach diff --git a/docker-compose-profiling.yml b/compose-profiling.yml similarity index 89% rename from docker-compose-profiling.yml rename to compose-profiling.yml index c0921d5d0a..c024d9fdcd 100644 --- a/docker-compose-profiling.yml +++ b/compose-profiling.yml @@ -10,6 +10,9 @@ services: - --feature-gates=service.profilesSupport privileged: true pid: "host" + environment: + - OTEL_COLLECTOR_HOST + - OTEL_COLLECTOR_PORT_GRPC volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - ./src/otel-collector/otelcol-ebpf-profiling.yml:/etc/ebpf-profiler-config.yaml:ro @@ -18,7 +21,7 @@ services: - /sys/fs/cgroup:/sys/fs/cgroup:ro - /proc:/proc:ro depends_on: - - firepit + - otel-collector firepit: image: ghcr.io/florianl/firepit:latest build: diff --git a/docker-compose.yml b/docker-compose.yml index 6e69ce3135..8985a8c5f0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -867,7 +867,10 @@ services: limits: memory: 200M restart: unless-stopped - command: [ "--config=/etc/otelcol-config.yml", "--config=/etc/otelcol-config-extras.yml" ] + command: + - --config=/etc/otelcol-config.yml + - --config=/etc/otelcol-config-extras.yml + - --feature-gates=service.profilesSupport user: 0:0 volumes: - ${HOST_FILESYSTEM}:/hostfs:ro diff --git a/src/otel-collector/otelcol-config.yml b/src/otel-collector/otelcol-config.yml index 4e11bf99a8..610fb79077 100644 --- a/src/otel-collector/otelcol-config.yml +++ b/src/otel-collector/otelcol-config.yml @@ -153,6 +153,10 @@ exporters: num_consumers: 10 queue_size: 1000 batch: + otlp/firepit: + endpoint: "firepit:4317" + tls: + insecure: true processors: memory_limiter: check_interval: 5s @@ -223,6 +227,10 @@ service: receivers: [otlp] processors: [resourcedetection, memory_limiter] exporters: [opensearch, debug] + profiles: + receivers: [otlp] + processors: [resourcedetection, memory_limiter] + exporters: [otlp/firepit] telemetry: metrics: readers: diff --git a/src/otel-collector/otelcol-ebpf-profiling.yml b/src/otel-collector/otelcol-ebpf-profiling.yml index d2c4a10017..58bbb0a24e 100644 --- a/src/otel-collector/otelcol-ebpf-profiling.yml +++ b/src/otel-collector/otelcol-ebpf-profiling.yml @@ -4,9 +4,6 @@ receivers: profiling: samples_per_second: 10 - reporter_interval: 10s - monitor_interval: 5s - off_cpu_threshold: 1.0 processors: resourcedetection: @@ -18,8 +15,8 @@ processors: action: upsert exporters: - otlp/firepit: - endpoint: firepit:4317 + otlp/collector: + endpoint: ${env:OTEL_COLLECTOR_HOST}:${env:OTEL_COLLECTOR_PORT_GRPC} tls: insecure: true @@ -28,4 +25,4 @@ service: profiles: receivers: [profiling] processors: [resourcedetection] - exporters: [otlp/firepit] + exporters: [otlp/collector] From 85b8cebfe9884cc7c52fe59136543342a30f0085 Mon Sep 17 00:00:00 2001 From: Donal O'Sullivan Date: Fri, 1 May 2026 10:09:41 +0100 Subject: [PATCH 08/15] Fix trailing whitespace in docker-compose --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8985a8c5f0..118865a98c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -868,7 +868,7 @@ services: memory: 200M restart: unless-stopped command: - - --config=/etc/otelcol-config.yml + - --config=/etc/otelcol-config.yml - --config=/etc/otelcol-config-extras.yml - --feature-gates=service.profilesSupport user: 0:0 From f32f7ee22aedf78bfccd081a2e356181e7f83e00 Mon Sep 17 00:00:00 2001 From: Donal O'Sullivan Date: Wed, 6 May 2026 09:30:26 +0100 Subject: [PATCH 09/15] Use prime number for ebpf samples per second to avoid over sampling --- src/otel-collector/otelcol-ebpf-profiling.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/otel-collector/otelcol-ebpf-profiling.yml b/src/otel-collector/otelcol-ebpf-profiling.yml index 58bbb0a24e..a0049c1923 100644 --- a/src/otel-collector/otelcol-ebpf-profiling.yml +++ b/src/otel-collector/otelcol-ebpf-profiling.yml @@ -3,7 +3,7 @@ receivers: profiling: - samples_per_second: 10 + samples_per_second: 19 processors: resourcedetection: From fd21fa0c4749cc78f0d4fd0c8dd68edcd4541cca Mon Sep 17 00:00:00 2001 From: Donal O'Sullivan Date: Wed, 6 May 2026 09:30:50 +0100 Subject: [PATCH 10/15] Use commit not latest for profiling compose --- compose-profiling.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose-profiling.yml b/compose-profiling.yml index c024d9fdcd..a89c1af74a 100644 --- a/compose-profiling.yml +++ b/compose-profiling.yml @@ -23,7 +23,7 @@ services: depends_on: - otel-collector firepit: - image: ghcr.io/florianl/firepit:latest + image: ghcr.io/florianl/firepit:6ea852a build: context: https://github.com/florianl/firepit.git container_name: firepit From 8e24601fffa2c3e02c260f4973d1d899510b3e29 Mon Sep 17 00:00:00 2001 From: Donal O'Sullivan Date: Wed, 6 May 2026 11:13:10 +0100 Subject: [PATCH 11/15] Remove entries from changlelog from bad merge conflict resolution --- CHANGELOG.md | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d86ef1674d..c77cdfbd5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,19 +10,6 @@ the release. * [profiling] Add profiling and use firepit as the backend to ingest profiles. This allows us to view profiles in the firepit webui. [#3333](https://github.com/open-telemetry/opentelemetry-demo/pull/3333) -* [frontend] fix: handle corrupted session data in localStorage - ([#3313](https://github.com/open-telemetry/opentelemetry-demo/pull/3313)) -* [docker] Podman doesn't support the tag feature of docker logs, - for the otel-demo to support podman we need to remove the tag from docker logs. - [#3304](https://github.com/open-telemetry/opentelemetry-demo/pull/3304) -* [payment] Fix `charge` span lifecycle and exception attribution: wrap charge - logic in `try/catch/finally` to ensure the span is always ended, record - exceptions on the `charge` span where they originate, and remove duplicate - `recordException` from the gRPC handler - ([#3276](https://github.com/open-telemetry/opentelemetry-demo/pull/3276)) -* [recommendation] Fix `recommendationCacheFailure` feature flag by - using `ListProducts` instead of `GetProduct` - ([#3260](https://github.com/open-telemetry/opentelemetry-demo/pull/3260)) * [telemetry-docs] Add a new service to provide telemetry documentation based on Weaver ([#2794](https://github.com/open-telemetry/opentelemetry-demo/pull/2794)) From d63e8503462194ae960d9ccc6c091a70dd3411ac Mon Sep 17 00:00:00 2001 From: Juliano Costa Date: Thu, 7 May 2026 18:04:16 +0200 Subject: [PATCH 12/15] Move Firepit to under Envoy --- .env | 6 +++- Makefile | 9 ++++++ compose-profiling.yml | 13 ++++++--- docker-compose.minimal.yml | 2 ++ docker-compose.yml | 2 ++ src/frontend-proxy/envoy.tmpl.yaml | 16 +++++++++++ src/otel-collector/otelcol-config.yml | 28 +++++++++++++------ src/otel-collector/otelcol-ebpf-profiling.yml | 21 +++++++------- 8 files changed, 74 insertions(+), 23 deletions(-) diff --git a/.env b/.env index 3cffea919a..09b87e02f9 100644 --- a/.env +++ b/.env @@ -9,7 +9,7 @@ OTEL_JAVA_AGENT_VERSION=2.25.0 OPENTELEMETRY_CPP_VERSION=1.24.0 # Dependent images -COLLECTOR_CONTRIB_IMAGE=ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.146.1 +COLLECTOR_CONTRIB_IMAGE=ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.151.0 FLAGD_IMAGE=ghcr.io/open-feature/flagd:v0.14.2 GRAFANA_IMAGE=grafana/grafana:13.0.1 JAEGERTRACING_IMAGE=quay.io/jaegertracing/jaeger:2.14.1 @@ -194,6 +194,10 @@ PROMETHEUS_PORT=9090 PROMETHEUS_HOST=prometheus PROMETHEUS_ADDR=${PROMETHEUS_HOST}:${PROMETHEUS_PORT} +# Firepit +FIREPIT_PORT=8080 +FIREPIT_HOST=firepit + # Telemetry Documentation TELEMETRY_DOCS_HOST=telemetry-docs TELEMETRY_DOCS_PORT=8000 diff --git a/Makefile b/Makefile index 34c4909ac2..b2ed324d90 100644 --- a/Makefile +++ b/Makefile @@ -259,3 +259,12 @@ build-react-native-android: .PHONY: start-profiling start-profiling: $(DOCKER_COMPOSE_CMD) $(DOCKER_COMPOSE_ENV) -f docker-compose.yml -f compose-profiling.yml up --force-recreate --remove-orphans --detach + @echo "" + @echo "OpenTelemetry Demo in profiling mode is running." + @echo "Go to http://localhost:8080 for the demo UI." + @echo "Go to http://localhost:8080/jaeger/ui for the Jaeger UI." + @echo "Go to http://localhost:8080/grafana/ for the Grafana UI." + @echo "Go to http://localhost:8080/loadgen/ for the Load Generator UI." + @echo "Go to http://localhost:8080/profiles/ for the Firepit UI." + @echo "Go to http://localhost:8080/telemetry/ for the Weaver generated telemetry documentation." + @echo "Go to https://opentelemetry.io/docs/demo/feature-flags/ to learn how to change feature flags." \ No newline at end of file diff --git a/compose-profiling.yml b/compose-profiling.yml index a89c1af74a..57e6f08e75 100644 --- a/compose-profiling.yml +++ b/compose-profiling.yml @@ -1,9 +1,14 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 +networks: + default: + name: opentelemetry-demo + driver: bridge + services: otel-ebpf-profiler: - image: otel/opentelemetry-collector-ebpf-profiler:0.147.0 + image: otel/opentelemetry-collector-ebpf-profiler:0.151.0 container_name: otel-ebpf-profiler command: - --config=/etc/ebpf-profiler-config.yaml @@ -24,8 +29,8 @@ services: - otel-collector firepit: image: ghcr.io/florianl/firepit:6ea852a - build: - context: https://github.com/florianl/firepit.git container_name: firepit ports: - - "${FIREPIT_PORT:-4040}:8080" + - "${FIREPIT_PORT}" + environment: + - BASE_PATH=/profiles diff --git a/docker-compose.minimal.yml b/docker-compose.minimal.yml index f2291fd6ca..5ba2756708 100644 --- a/docker-compose.minimal.yml +++ b/docker-compose.minimal.yml @@ -284,6 +284,8 @@ services: - ENVOY_PORT - ENVOY_ADMIN_PORT - ENVOY_ADDR + - FIREPIT_HOST + - FIREPIT_PORT - FLAGD_HOST - FLAGD_PORT - FLAGD_UI_HOST diff --git a/docker-compose.yml b/docker-compose.yml index 118865a98c..b2a86a7c8b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -360,6 +360,8 @@ services: - ENVOY_PORT - ENVOY_ADDR - ENVOY_ADMIN_PORT + - FIREPIT_HOST + - FIREPIT_PORT - FLAGD_HOST - FLAGD_PORT - FLAGD_UI_HOST diff --git a/src/frontend-proxy/envoy.tmpl.yaml b/src/frontend-proxy/envoy.tmpl.yaml index 7a15267540..68f1d39f23 100644 --- a/src/frontend-proxy/envoy.tmpl.yaml +++ b/src/frontend-proxy/envoy.tmpl.yaml @@ -65,6 +65,10 @@ static_resources: prefix_rewrite: "/" upgrade_configs: - upgrade_type: websocket + - match: { path: "/profiles" } + redirect: { path_redirect: "/profiles/" } + - match: { prefix: "/profiles/" } + route: { cluster: profiles } - match: { prefix: "/" } # Default/catch-all route - keep last since prefix:"/" matches everything route: { cluster: frontend } http_filters: @@ -258,6 +262,18 @@ static_resources: socket_address: address: ${TELEMETRY_DOCS_HOST} port_value: ${TELEMETRY_DOCS_PORT} + - name: profiles + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: profiles + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: ${FIREPIT_HOST} + port_value: ${FIREPIT_PORT} admin: address: socket_address: diff --git a/src/otel-collector/otelcol-config.yml b/src/otel-collector/otelcol-config.yml index 610fb79077..da9d161bc1 100644 --- a/src/otel-collector/otelcol-config.yml +++ b/src/otel-collector/otelcol-config.yml @@ -12,7 +12,7 @@ receivers: allowed_origins: - "http://*" - "https://*" - httpcheck/frontend-proxy: + http_check/frontend-proxy: targets: - endpoint: http://${env:FRONTEND_PROXY_ADDR} nginx: @@ -58,7 +58,7 @@ receivers: username: "valkey" collection_interval: 10s # Host metrics - hostmetrics: + host_metrics: root_path: /hostfs scrapers: cpu: @@ -153,7 +153,7 @@ exporters: num_consumers: 10 queue_size: 1000 batch: - otlp/firepit: + otlp_grpc/firepit: endpoint: "firepit:4317" tls: insecure: true @@ -164,6 +164,18 @@ processors: spike_limit_percentage: 25 resourcedetection: detectors: [env, docker, system] + filter/profiles_exclude_system_processes: + error_mode: ignore + profiles: + profile: + - 'IsMatch(resource.attributes["service.name"], "containerd-shim-.*")' + transform/sanitize_profiles: + error_mode: ignore + profile_statements: + - context: resource + statements: + - delete_key(attributes, "process.executable.path") + - delete_key(attributes, "container.id") transform/sanitize_spans: error_mode: ignore trace_statements: @@ -211,16 +223,16 @@ processors: - set_semconv_span_name("1.37.0", "original_span_name") connectors: - spanmetrics: + span_metrics: service: pipelines: traces: receivers: [otlp] processors: [resourcedetection, memory_limiter, transform/sanitize_spans] - exporters: [otlp_grpc/jaeger, debug, spanmetrics] + exporters: [otlp_grpc/jaeger, debug, span_metrics] metrics: - receivers: [docker_stats, httpcheck/frontend-proxy, hostmetrics, nginx, otlp, postgresql, redis, spanmetrics, kafkametrics] + receivers: [docker_stats, http_check/frontend-proxy, host_metrics, nginx, otlp, postgresql, redis, span_metrics, kafkametrics] processors: [resourcedetection, memory_limiter] exporters: [otlp_http/prometheus, debug] logs: @@ -229,8 +241,8 @@ service: exporters: [opensearch, debug] profiles: receivers: [otlp] - processors: [resourcedetection, memory_limiter] - exporters: [otlp/firepit] + processors: [filter/profiles_exclude_system_processes, transform/sanitize_profiles, memory_limiter] + exporters: [otlp_grpc/firepit, debug] telemetry: metrics: readers: diff --git a/src/otel-collector/otelcol-ebpf-profiling.yml b/src/otel-collector/otelcol-ebpf-profiling.yml index a0049c1923..e7461e549f 100644 --- a/src/otel-collector/otelcol-ebpf-profiling.yml +++ b/src/otel-collector/otelcol-ebpf-profiling.yml @@ -4,18 +4,19 @@ receivers: profiling: samples_per_second: 19 + include_env_vars: "OTEL_SERVICE_NAME" processors: - resourcedetection: - detectors: [docker] - resource: - attributes: - - key: service.name - from_attribute: container.name - action: upsert + transform/set_service_name: + error_mode: ignore + profile_statements: + - context: resource + statements: + - set(attributes["service.name"], attributes["process.environment_variable.OTEL_SERVICE_NAME"]) where attributes["process.environment_variable.OTEL_SERVICE_NAME"] != nil + - delete_key(attributes, "process.environment_variable.OTEL_SERVICE_NAME") exporters: - otlp/collector: + otlp_grpc/collector: endpoint: ${env:OTEL_COLLECTOR_HOST}:${env:OTEL_COLLECTOR_PORT_GRPC} tls: insecure: true @@ -24,5 +25,5 @@ service: pipelines: profiles: receivers: [profiling] - processors: [resourcedetection] - exporters: [otlp/collector] + processors: [transform/set_service_name] + exporters: [otlp_grpc/collector] From cfe26cd3a043e76474b0d48e8135df9d63aaf0eb Mon Sep 17 00:00:00 2001 From: Juliano Costa Date: Mon, 11 May 2026 17:36:03 +0200 Subject: [PATCH 13/15] Bump firepit and remove filters --- compose-profiling.yml | 2 +- src/otel-collector/otelcol-config.yml | 15 +++------------ 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/compose-profiling.yml b/compose-profiling.yml index 57e6f08e75..b3cae4332e 100644 --- a/compose-profiling.yml +++ b/compose-profiling.yml @@ -28,7 +28,7 @@ services: depends_on: - otel-collector firepit: - image: ghcr.io/florianl/firepit:6ea852a + image: ghcr.io/florianl/firepit:df1dcbe container_name: firepit ports: - "${FIREPIT_PORT}" diff --git a/src/otel-collector/otelcol-config.yml b/src/otel-collector/otelcol-config.yml index 12e6a5db54..c49a3c7e0e 100644 --- a/src/otel-collector/otelcol-config.yml +++ b/src/otel-collector/otelcol-config.yml @@ -149,9 +149,6 @@ exporters: tls: insecure: true sending_queue: - # Explicitly set due to bug: https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/45016 - num_consumers: 10 - queue_size: 1000 batch: otlp_grpc/firepit: endpoint: "firepit:4317" @@ -164,18 +161,11 @@ processors: spike_limit_percentage: 25 resourcedetection: detectors: [env, docker, system] - filter/profiles_exclude_system_processes: + filter/sanitize_profiles: error_mode: ignore profiles: profile: - 'IsMatch(resource.attributes["service.name"], "containerd-shim-.*")' - transform/sanitize_profiles: - error_mode: ignore - profile_statements: - - context: resource - statements: - - delete_key(attributes, "process.executable.path") - - delete_key(attributes, "container.id") transform/sanitize_logs: error_mode: ignore log_statements: @@ -186,6 +176,7 @@ processors: # Upstream Issue: https://github.com/open-telemetry/opentelemetry-collector/issues/15221 - set(attributes["otelcol_signal"], attributes["otelcol.signal"]) where attributes["otelcol.signal"] != nil and not IsMap(attributes["otelcol.signal"]) - delete_key(attributes, "otelcol.signal") where attributes["otelcol_signal"] != nil + transform/sanitize_spans: error_mode: ignore trace_statements: @@ -251,7 +242,7 @@ service: exporters: [opensearch, debug] profiles: receivers: [otlp] - processors: [filter/profiles_exclude_system_processes, transform/sanitize_profiles, memory_limiter] + processors: [filter/sanitize_profiles, memory_limiter] exporters: [otlp_grpc/firepit, debug] telemetry: metrics: From 0cea1312bc62d5c4fe973fcc8eba538bd9c51d59 Mon Sep 17 00:00:00 2001 From: Juliano Costa Date: Mon, 11 May 2026 17:42:46 +0200 Subject: [PATCH 14/15] line break --- src/otel-collector/otelcol-config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/otel-collector/otelcol-config.yml b/src/otel-collector/otelcol-config.yml index c49a3c7e0e..87e8b3e014 100644 --- a/src/otel-collector/otelcol-config.yml +++ b/src/otel-collector/otelcol-config.yml @@ -176,7 +176,6 @@ processors: # Upstream Issue: https://github.com/open-telemetry/opentelemetry-collector/issues/15221 - set(attributes["otelcol_signal"], attributes["otelcol.signal"]) where attributes["otelcol.signal"] != nil and not IsMap(attributes["otelcol.signal"]) - delete_key(attributes, "otelcol.signal") where attributes["otelcol_signal"] != nil - transform/sanitize_spans: error_mode: ignore trace_statements: From 4c351ab0f84396aa25e15931ce7d87d9c773ea54 Mon Sep 17 00:00:00 2001 From: Donal O'Sullivan Date: Fri, 15 May 2026 15:51:00 +0100 Subject: [PATCH 15/15] Add profiling feature gate to compose minimal --- docker-compose.minimal.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker-compose.minimal.yml b/docker-compose.minimal.yml index 5ba2756708..05e4d28431 100644 --- a/docker-compose.minimal.yml +++ b/docker-compose.minimal.yml @@ -718,7 +718,10 @@ services: limits: memory: 200M restart: unless-stopped - command: [ "--config=/etc/otelcol-config.yml", "--config=/etc/otelcol-config-extras.yml" ] + command: + - --config=/etc/otelcol-config.yml + - --config=/etc/otelcol-config-extras.yml + - --feature-gates=service.profilesSupport user: 0:0 volumes: - ${HOST_FILESYSTEM}:/hostfs:ro