Profiling with firepit

.env

@@ -9,7 +9,7 @@ OTEL_JAVA_AGENT_VERSION=2.25.0
 OPENTELEMETRY_CPP_VERSION=1.24.0
 
 # Dependent images
-COLLECTOR_CONTRIB_IMAGE=ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.146.1
+COLLECTOR_CONTRIB_IMAGE=ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.151.0
 FLAGD_IMAGE=ghcr.io/open-feature/flagd:v0.14.2
 GRAFANA_IMAGE=grafana/grafana:13.0.1
 JAEGERTRACING_IMAGE=quay.io/jaegertracing/jaeger:2.14.1
@@ -194,6 +194,10 @@ PROMETHEUS_PORT=9090
 PROMETHEUS_HOST=prometheus
 PROMETHEUS_ADDR=${PROMETHEUS_HOST}:${PROMETHEUS_PORT}
 
+# Firepit
+FIREPIT_PORT=8080
+FIREPIT_HOST=firepit
+
 # Telemetry Documentation
 TELEMETRY_DOCS_HOST=telemetry-docs
 TELEMETRY_DOCS_PORT=8000

CHANGELOG.md

@@ -7,10 +7,6 @@ the release.
 
 ## Unreleased
 
-* [collector] Add `transform/sanitize_logs` processor to work around
-  `otelcol.signal` scope attribute conflict with `otelcol.signal.output`
-  that causes OpenSearch/Elasticsearch mapping failures
-  ([#3321](https://github.com/open-telemetry/opentelemetry-demo/pull/3321))
 * [telemetry-docs] Add a new service to provide telemetry documentation based
   on Weaver
   ([#2794](https://github.com/open-telemetry/opentelemetry-demo/pull/2794))
@@ -80,6 +76,13 @@ the release.
   [#3307](https://github.com/open-telemetry/opentelemetry-demo/pull/3307)
 * [frontend] fix: handle corrupted session data in localStorage
   ([#3313](https://github.com/open-telemetry/opentelemetry-demo/pull/3313))
+* [collector] Add `transform/sanitize_logs` processor to work around
+  `otelcol.signal` scope attribute conflict with `otelcol.signal.output`
+  that causes OpenSearch/Elasticsearch mapping failures
+  ([#3321](https://github.com/open-telemetry/opentelemetry-demo/pull/3321))
+* [profiling] Add profiling and use firepit as the backend to ingest profiles.
+  This allows us to view profiles in the firepit webui.
+  [#3333](https://github.com/open-telemetry/opentelemetry-demo/pull/3333)
 
 ## 2.2.0
 

Makefile

@@ -255,3 +255,16 @@ endif
 .PHONY: build-react-native-android
 build-react-native-android:
 	$(DOCKER_CMD) build -f src/react-native-app/android.Dockerfile --platform=linux/amd64 --output=. src/react-native-app
+
+.PHONY: start-profiling
+start-profiling:
+	$(DOCKER_COMPOSE_CMD) $(DOCKER_COMPOSE_ENV) -f docker-compose.yml -f compose-profiling.yml up --force-recreate --remove-orphans --detach
+	@echo ""
+	@echo "OpenTelemetry Demo in profiling mode is running."
+	@echo "Go to http://localhost:8080 for the demo UI."
+	@echo "Go to http://localhost:8080/jaeger/ui for the Jaeger UI."
+	@echo "Go to http://localhost:8080/grafana/ for the Grafana UI."
+	@echo "Go to http://localhost:8080/loadgen/ for the Load Generator UI."
+	@echo "Go to http://localhost:8080/profiles/ for the Firepit UI."
+	@echo "Go to http://localhost:8080/telemetry/ for the Weaver generated telemetry documentation."
+	@echo "Go to https://opentelemetry.io/docs/demo/feature-flags/ to learn how to change feature flags."

compose-profiling.yml

@@ -0,0 +1,36 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+networks:
+  default:
+    name: opentelemetry-demo
+    driver: bridge
+
+services:
+  otel-ebpf-profiler:
+    image: otel/opentelemetry-collector-ebpf-profiler:0.151.0
+    container_name: otel-ebpf-profiler
+    command:
+      - --config=/etc/ebpf-profiler-config.yaml
+      - --feature-gates=service.profilesSupport
+    privileged: true
+    pid: "host"
+    environment:
+      - OTEL_COLLECTOR_HOST
+      - OTEL_COLLECTOR_PORT_GRPC
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ./src/otel-collector/otelcol-ebpf-profiling.yml:/etc/ebpf-profiler-config.yaml:ro
+      - /sys/kernel/debug:/sys/kernel/debug
+      - /sys/kernel/tracing:/sys/kernel/tracing
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+      - /proc:/proc:ro
+    depends_on:
+      - otel-collector
+  firepit:
+    image: ghcr.io/florianl/firepit:df1dcbe
+    container_name: firepit
+    ports:
+      - "${FIREPIT_PORT}"
+    environment:
+      - BASE_PATH=/profiles

docker-compose.minimal.yml

@@ -284,6 +284,8 @@ services:
       - ENVOY_PORT
       - ENVOY_ADMIN_PORT
       - ENVOY_ADDR
+      - FIREPIT_HOST
+      - FIREPIT_PORT
       - FLAGD_HOST
       - FLAGD_PORT
       - FLAGD_UI_HOST
@@ -716,7 +718,10 @@ services:
         limits:
           memory: 200M
     restart: unless-stopped
-    command: [ "--config=/etc/otelcol-config.yml", "--config=/etc/otelcol-config-extras.yml" ]
+    command:
+      - --config=/etc/otelcol-config.yml
+      - --config=/etc/otelcol-config-extras.yml
+      - --feature-gates=service.profilesSupport
     user: 0:0
     volumes:
       - ${HOST_FILESYSTEM}:/hostfs:ro

docker-compose.yml

@@ -360,6 +360,8 @@ services:
       - ENVOY_PORT
       - ENVOY_ADDR
       - ENVOY_ADMIN_PORT
+      - FIREPIT_HOST
+      - FIREPIT_PORT
       - FLAGD_HOST
       - FLAGD_PORT
       - FLAGD_UI_HOST
@@ -867,7 +869,10 @@ services:
         limits:
           memory: 200M
     restart: unless-stopped
-    command: [ "--config=/etc/otelcol-config.yml", "--config=/etc/otelcol-config-extras.yml" ]
+    command:
+      - --config=/etc/otelcol-config.yml
+      - --config=/etc/otelcol-config-extras.yml
+      - --feature-gates=service.profilesSupport
     user: 0:0
     volumes:
       - ${HOST_FILESYSTEM}:/hostfs:ro

src/frontend-proxy/envoy.tmpl.yaml

@@ -65,6 +65,10 @@ static_resources:
                             prefix_rewrite: "/"
                             upgrade_configs:
                               - upgrade_type: websocket
+                        - match: { path: "/profiles" }
+                          redirect: { path_redirect: "/profiles/" }
+                        - match: { prefix: "/profiles/" }
+                          route: { cluster: profiles }
                         - match: { prefix: "/" }  # Default/catch-all route - keep last since prefix:"/" matches everything
                           route: { cluster: frontend }
                 http_filters:
@@ -258,6 +262,18 @@ static_resources:
                     socket_address:
                       address: ${TELEMETRY_DOCS_HOST}
                       port_value: ${TELEMETRY_DOCS_PORT}
+    - name: profiles
+      type: STRICT_DNS
+      lb_policy: ROUND_ROBIN
+      load_assignment:
+        cluster_name: profiles
+        endpoints:
+          - lb_endpoints:
+              - endpoint:
+                  address:
+                    socket_address:
+                      address: ${FIREPIT_HOST}
+                      port_value: ${FIREPIT_PORT}
 admin:
   address:
     socket_address:

src/otel-collector/otelcol-config.yml

@@ -12,7 +12,7 @@ receivers:
           allowed_origins:
             - "http://*"
             - "https://*"
-  httpcheck/frontend-proxy:
+  http_check/frontend-proxy:
     targets:
       - endpoint: http://${env:FRONTEND_PROXY_ADDR}
   nginx:
@@ -58,7 +58,7 @@ receivers:
     username: "valkey"
     collection_interval: 10s
   # Host metrics
-  hostmetrics:
+  host_metrics:
     root_path: /hostfs
     scrapers:
       cpu:
@@ -149,17 +149,23 @@ exporters:
       tls:
         insecure: true
     sending_queue:
-      # Explicitly set due to bug: https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/45016
-      num_consumers: 10
-      queue_size: 1000
       batch:
+  otlp_grpc/firepit:
+    endpoint: "firepit:4317"
+    tls:
+      insecure: true
 processors:
   memory_limiter:
     check_interval: 5s
     limit_percentage: 80
     spike_limit_percentage: 25
   resourcedetection:
     detectors: [env, docker, system]
+  filter/sanitize_profiles:
+    error_mode: ignore
+    profiles:
+      profile:
+        - 'IsMatch(resource.attributes["service.name"], "containerd-shim-.*")'
   transform/sanitize_logs:
     error_mode: ignore
     log_statements:
@@ -217,22 +223,26 @@ processors:
           - set_semconv_span_name("1.37.0", "original_span_name")
 
 connectors:
-  spanmetrics:
+  span_metrics:
 
 service:
   pipelines:
     traces:
       receivers: [otlp]
       processors: [resourcedetection, memory_limiter, transform/sanitize_spans]
-      exporters: [otlp_grpc/jaeger, debug, spanmetrics]
+      exporters: [otlp_grpc/jaeger, debug, span_metrics]
     metrics:
-      receivers: [docker_stats, httpcheck/frontend-proxy, hostmetrics, nginx, otlp, postgresql, redis, spanmetrics, kafkametrics]
+      receivers: [docker_stats, http_check/frontend-proxy, host_metrics, nginx, otlp, postgresql, redis, span_metrics, kafkametrics]
       processors: [resourcedetection, memory_limiter]
       exporters: [otlp_http/prometheus, debug]
     logs:
       receivers: [otlp]
       processors: [resourcedetection, memory_limiter, transform/sanitize_logs]
       exporters: [opensearch, debug]
+    profiles:
+      receivers: [otlp]
+      processors: [filter/sanitize_profiles, memory_limiter]
+      exporters: [otlp_grpc/firepit, debug]
   telemetry:
     metrics:
       readers:

src/otel-collector/otelcol-ebpf-profiling.yml

@@ -0,0 +1,29 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+receivers:
+  profiling:
+    samples_per_second: 19
+    include_env_vars: "OTEL_SERVICE_NAME"
+
+processors:
+  transform/set_service_name:
+    error_mode: ignore
+    profile_statements:
+      - context: resource
+        statements:
+          - set(attributes["service.name"], attributes["process.environment_variable.OTEL_SERVICE_NAME"]) where attributes["process.environment_variable.OTEL_SERVICE_NAME"] != nil
+          - delete_key(attributes, "process.environment_variable.OTEL_SERVICE_NAME")
+
+exporters:
+  otlp_grpc/collector:
+    endpoint: ${env:OTEL_COLLECTOR_HOST}:${env:OTEL_COLLECTOR_PORT_GRPC}
+    tls:
+      insecure: true
+
+service:
+  pipelines:
+    profiles:
+      receivers: [profiling]
+      processors: [transform/set_service_name]
+      exporters: [otlp_grpc/collector]