From 589a2d48ea0c91119dc908ff0c9600bc793eb217 Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Fri, 8 May 2026 15:17:32 -0700 Subject: [PATCH 01/13] c --- rust/otap-dataflow/cross-arch-build.sh | 2 +- rust/otap-dataflow/glibc.Dockerfile | 47 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 rust/otap-dataflow/glibc.Dockerfile diff --git a/rust/otap-dataflow/cross-arch-build.sh b/rust/otap-dataflow/cross-arch-build.sh index d2d915aea2..7276c06576 100755 --- a/rust/otap-dataflow/cross-arch-build.sh +++ b/rust/otap-dataflow/cross-arch-build.sh @@ -1,7 +1,7 @@ set -exu if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then - RUST_TARGET=x86_64-unknown-linux-musl + RUST_TARGET=x86_64-unknown-linux-gnu apt-get update && apt-get install -y musl-tools export CC_x86_64_unknown_linux_musl=musl-gcc if [ "${TARGETPLATFORM}" != "${BUILDPLATFORM}" ]; then diff --git a/rust/otap-dataflow/glibc.Dockerfile b/rust/otap-dataflow/glibc.Dockerfile new file mode 100644 index 0000000000..c96772d665 --- /dev/null +++ b/rust/otap-dataflow/glibc.Dockerfile @@ -0,0 +1,47 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# This Dockerfile builds the otap-dataflow engine from the source in this +# directory. It relies on protobuf files in the root /proto directory. +# In order to include these in the build context, and to avoid conflicting +# with the arrow enabled go collector build files in the repo root, the image +# must be built with additional --build-context flags as follows: +# docker build --build-context otel-arrow=../../ -f Dockerfile -t df_engine . +FROM --platform=$BUILDPLATFORM docker.io/rust:1.95@sha256:4a7e3a0c309c9bab658e469f842711bd595fae484936bc5d605e08ca0c631bf4 AS dataflow-build +WORKDIR /build/rust/dataflow +ARG BUILDPLATFORM +ARG TARGETPLATFORM + +RUN apt-get update && apt-get install -y protobuf-compiler + +# The build process relies on these libraries outside of the current context. +# Build command from the otap-dataflow directory: +# docker build --build-context otel-arrow=../../ -f Dockerfile -t df_engine . +COPY --from=otel-arrow /proto/opentelemetry/proto /build/proto/opentelemetry/proto +COPY --from=otel-arrow /proto/opentelemetry-proto /build/proto/opentelemetry-proto +COPY --from=otel-arrow /rust/experimental /build/rust/experimental +COPY --from=otel-arrow /THIRD_PARTY_NOTICES.txt /build/THIRD_PARTY_NOTICES.txt + +COPY . /build/rust/dataflow/. + +# RUSTFLAGS can be used to pass argument to rustc e.g. +# --build-arg RUSTFLAGS="-C target-feature=-gfni" +ARG RUSTFLAGS="-C target-cpu=native" +ENV RUSTFLAGS=${RUSTFLAGS} + +# These are used to enable features and are passed to cargo's --features argument +# e.g. --build-arg FEATURES="azure" +ARG FEATURES +ENV FEATURES=${FEATURES} + +# Build dataflow engine +RUN ./cross-arch-build.sh + +# Runtime image +FROM gcr.io/distroless/cc-debian13:nonroot +LABEL maintainer="The OpenTelemetry Authors" + +WORKDIR /home/nonroot +COPY --from=dataflow-build --chown=nonroot:nonroot /build/rust/dataflow/df_engine . +USER nonroot +ENTRYPOINT ["/home/nonroot/df_engine"] From 18d21ec984a0d2e93c458125500c3fe73d1f3331 Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Fri, 8 May 2026 15:39:53 -0700 Subject: [PATCH 02/13] c --- rust/otap-dataflow/.cargo/config.toml | 4 +- rust/otap-dataflow/Dockerfile | 24 ++++------- rust/otap-dataflow/cross-arch-build.sh | 58 ++++++-------------------- rust/otap-dataflow/glibc.Dockerfile | 47 --------------------- 4 files changed, 21 insertions(+), 112 deletions(-) delete mode 100644 rust/otap-dataflow/glibc.Dockerfile diff --git a/rust/otap-dataflow/.cargo/config.toml b/rust/otap-dataflow/.cargo/config.toml index d431140798..03b6132779 100644 --- a/rust/otap-dataflow/.cargo/config.toml +++ b/rust/otap-dataflow/.cargo/config.toml @@ -1,10 +1,10 @@ [alias] xtask = "run --package xtask --bin xtask --" -[target.aarch64-unknown-linux-musl] +[target.aarch64-unknown-linux-gnu] linker = "aarch64-linux-gnu-gcc" rustflags = ["-C", "target-cpu=native"] -[target.x86_64-unknown-linux-musl] +[target.x86_64-unknown-linux-gnu] linker = "x86_64-linux-gnu-gcc" rustflags = ["-C", "target-cpu=native"] \ No newline at end of file diff --git a/rust/otap-dataflow/Dockerfile b/rust/otap-dataflow/Dockerfile index b621c2f1ee..c96772d665 100644 --- a/rust/otap-dataflow/Dockerfile +++ b/rust/otap-dataflow/Dockerfile @@ -7,7 +7,7 @@ # with the arrow enabled go collector build files in the repo root, the image # must be built with additional --build-context flags as follows: # docker build --build-context otel-arrow=../../ -f Dockerfile -t df_engine . -FROM --platform=$BUILDPLATFORM docker.io/rust:1.95@sha256:a9cfb755b33f5bb872610cbdb25da61f527416b28fc9c052bbce4bef93e7799a AS dataflow-build +FROM --platform=$BUILDPLATFORM docker.io/rust:1.95@sha256:4a7e3a0c309c9bab658e469f842711bd595fae484936bc5d605e08ca0c631bf4 AS dataflow-build WORKDIR /build/rust/dataflow ARG BUILDPLATFORM ARG TARGETPLATFORM @@ -37,21 +37,11 @@ ENV FEATURES=${FEATURES} # Build dataflow engine RUN ./cross-arch-build.sh -# The runtime image -FROM docker.io/alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 +# Runtime image +FROM gcr.io/distroless/cc-debian13:nonroot LABEL maintainer="The OpenTelemetry Authors" -# Passing --build-arg BUILD=1 adds additional debugging features to the image. -ARG DEBUG="0" -RUN if [ "$DEBUG" = "1" ]; then apk add --no-cache gdb; fi - -RUN addgroup dataflow \ - && adduser \ - --ingroup dataflow \ - --disabled-password \ - dataflow -WORKDIR /home/dataflow - -COPY --from=dataflow-build --chown=dataflow:dataflow /build/rust/dataflow/df_engine . -USER dataflow -ENTRYPOINT ["/home/dataflow/df_engine"] +WORKDIR /home/nonroot +COPY --from=dataflow-build --chown=nonroot:nonroot /build/rust/dataflow/df_engine . +USER nonroot +ENTRYPOINT ["/home/nonroot/df_engine"] diff --git a/rust/otap-dataflow/cross-arch-build.sh b/rust/otap-dataflow/cross-arch-build.sh index 7276c06576..275196b0a1 100755 --- a/rust/otap-dataflow/cross-arch-build.sh +++ b/rust/otap-dataflow/cross-arch-build.sh @@ -2,55 +2,21 @@ set -exu if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then RUST_TARGET=x86_64-unknown-linux-gnu - apt-get update && apt-get install -y musl-tools - export CC_x86_64_unknown_linux_musl=musl-gcc if [ "${TARGETPLATFORM}" != "${BUILDPLATFORM}" ]; then - apt-get update && apt-get install -y gcc-x86-64-linux-gnu + apt-get update && apt-get install -y gcc-x86-64-linux-gnu g++-x86-64-linux-gnu + export CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc + export CXX_x86_64_unknown_linux_gnu=x86_64-linux-gnu-g++ + export AR_x86_64_unknown_linux_gnu=x86_64-linux-gnu-ar + export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc fi elif [ "${TARGETPLATFORM}" = "linux/arm64" ]; then - RUST_TARGET=aarch64-unknown-linux-musl - if [ "${BUILDPLATFORM}" = "linux/arm64" ]; then - # Native arm64 builder: use the musl.cc arm64-hosted native toolchain. - # musl-tools/musl-gcc is avoided because its unprefixed compiler name causes - # jemalloc's configure to misdetect the build as cross-compilation - # (build=aarch64-linux-gnu vs host=aarch64-linux-musl), which breaks atomic - # detection. The musl.cc native toolchain provides a properly-prefixed - # aarch64-linux-musl-gcc that runs natively on arm64 with musl headers. - MUSL_NATIVE_TARBALL=aarch64-linux-musl-native.tgz - MUSL_NATIVE_SHA256=daf336cafa2c3c7daf42f6a46edc960f10a181fcf15ab9f1c43b192e8ad2a069 - MUSL_NATIVE_DIR=/opt/aarch64-linux-musl-native - curl -sSfL "https://musl.cc/${MUSL_NATIVE_TARBALL}" -o "/tmp/${MUSL_NATIVE_TARBALL}" - echo "${MUSL_NATIVE_SHA256} /tmp/${MUSL_NATIVE_TARBALL}" | sha256sum -c - - tar xz -C /opt -f "/tmp/${MUSL_NATIVE_TARBALL}" - rm "/tmp/${MUSL_NATIVE_TARBALL}" - # The native tarball ships aarch64-linux-musl-gcc-ar but not aarch64-linux-musl-ar. - # cmake-based crates (e.g. zstd-sys) look for the latter by name, so symlink it. - ln -s "${MUSL_NATIVE_DIR}/bin/aarch64-linux-musl-gcc-ar" "${MUSL_NATIVE_DIR}/bin/aarch64-linux-musl-ar" - export CC_aarch64_unknown_linux_musl=${MUSL_NATIVE_DIR}/bin/aarch64-linux-musl-gcc - export CXX_aarch64_unknown_linux_musl=${MUSL_NATIVE_DIR}/bin/aarch64-linux-musl-g++ - export AR_aarch64_unknown_linux_musl=${MUSL_NATIVE_DIR}/bin/aarch64-linux-musl-ar - export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=${MUSL_NATIVE_DIR}/bin/aarch64-linux-musl-gcc - elif [ "${BUILDPLATFORM}" = "linux/amd64" ]; then - # Cross-compile (amd64 -> arm64): Debian's gcc-aarch64-linux-gnu uses glibc - # system headers; glibc >= 2.38 maps _GNU_SOURCE to _ISOC23_SOURCE, which - # redirects strtol/sscanf to __isoc23_strtol/__isoc23_sscanf. Those C23 - # symbols don't exist in musl libc, causing link failures in C dependencies - # that define _GNU_SOURCE (e.g. aws-lc-sys/bcm.c). Use the musl.cc - # x86_64-hosted cross-toolchain which ships its own musl headers instead. - MUSL_CROSS_TARBALL=aarch64-linux-musl-cross.tgz - MUSL_CROSS_SHA256=c909817856d6ceda86aa510894fa3527eac7989f0ef6e87b5721c58737a06c38 - MUSL_CROSS_DIR=/opt/aarch64-linux-musl-cross - curl -sSfL "https://musl.cc/${MUSL_CROSS_TARBALL}" -o "/tmp/${MUSL_CROSS_TARBALL}" - echo "${MUSL_CROSS_SHA256} /tmp/${MUSL_CROSS_TARBALL}" | sha256sum -c - - tar xz -C /opt -f "/tmp/${MUSL_CROSS_TARBALL}" - rm "/tmp/${MUSL_CROSS_TARBALL}" - export CC_aarch64_unknown_linux_musl=${MUSL_CROSS_DIR}/bin/aarch64-linux-musl-gcc - export CXX_aarch64_unknown_linux_musl=${MUSL_CROSS_DIR}/bin/aarch64-linux-musl-g++ - export AR_aarch64_unknown_linux_musl=${MUSL_CROSS_DIR}/bin/aarch64-linux-musl-ar - export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=${MUSL_CROSS_DIR}/bin/aarch64-linux-musl-gcc - else - echo "Unsupported build platform for arm64 target: ${BUILDPLATFORM}" - exit 1 + RUST_TARGET=aarch64-unknown-linux-gnu + if [ "${TARGETPLATFORM}" != "${BUILDPLATFORM}" ]; then + apt-get update && apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu + export CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc + export CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ + export AR_aarch64_unknown_linux_gnu=aarch64-linux-gnu-ar + export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc fi else echo "Unsupported target platform: ${TARGETPLATFORM}" diff --git a/rust/otap-dataflow/glibc.Dockerfile b/rust/otap-dataflow/glibc.Dockerfile deleted file mode 100644 index c96772d665..0000000000 --- a/rust/otap-dataflow/glibc.Dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -# This Dockerfile builds the otap-dataflow engine from the source in this -# directory. It relies on protobuf files in the root /proto directory. -# In order to include these in the build context, and to avoid conflicting -# with the arrow enabled go collector build files in the repo root, the image -# must be built with additional --build-context flags as follows: -# docker build --build-context otel-arrow=../../ -f Dockerfile -t df_engine . -FROM --platform=$BUILDPLATFORM docker.io/rust:1.95@sha256:4a7e3a0c309c9bab658e469f842711bd595fae484936bc5d605e08ca0c631bf4 AS dataflow-build -WORKDIR /build/rust/dataflow -ARG BUILDPLATFORM -ARG TARGETPLATFORM - -RUN apt-get update && apt-get install -y protobuf-compiler - -# The build process relies on these libraries outside of the current context. -# Build command from the otap-dataflow directory: -# docker build --build-context otel-arrow=../../ -f Dockerfile -t df_engine . -COPY --from=otel-arrow /proto/opentelemetry/proto /build/proto/opentelemetry/proto -COPY --from=otel-arrow /proto/opentelemetry-proto /build/proto/opentelemetry-proto -COPY --from=otel-arrow /rust/experimental /build/rust/experimental -COPY --from=otel-arrow /THIRD_PARTY_NOTICES.txt /build/THIRD_PARTY_NOTICES.txt - -COPY . /build/rust/dataflow/. - -# RUSTFLAGS can be used to pass argument to rustc e.g. -# --build-arg RUSTFLAGS="-C target-feature=-gfni" -ARG RUSTFLAGS="-C target-cpu=native" -ENV RUSTFLAGS=${RUSTFLAGS} - -# These are used to enable features and are passed to cargo's --features argument -# e.g. --build-arg FEATURES="azure" -ARG FEATURES -ENV FEATURES=${FEATURES} - -# Build dataflow engine -RUN ./cross-arch-build.sh - -# Runtime image -FROM gcr.io/distroless/cc-debian13:nonroot -LABEL maintainer="The OpenTelemetry Authors" - -WORKDIR /home/nonroot -COPY --from=dataflow-build --chown=nonroot:nonroot /build/rust/dataflow/df_engine . -USER nonroot -ENTRYPOINT ["/home/nonroot/df_engine"] From 98a1500672edb2a51d4e591575a72ae6c87aee2a Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Fri, 8 May 2026 15:44:57 -0700 Subject: [PATCH 03/13] c --- rust/otap-dataflow/cross-arch-build.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/rust/otap-dataflow/cross-arch-build.sh b/rust/otap-dataflow/cross-arch-build.sh index 275196b0a1..6043653ff0 100755 --- a/rust/otap-dataflow/cross-arch-build.sh +++ b/rust/otap-dataflow/cross-arch-build.sh @@ -4,19 +4,11 @@ if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then RUST_TARGET=x86_64-unknown-linux-gnu if [ "${TARGETPLATFORM}" != "${BUILDPLATFORM}" ]; then apt-get update && apt-get install -y gcc-x86-64-linux-gnu g++-x86-64-linux-gnu - export CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc - export CXX_x86_64_unknown_linux_gnu=x86_64-linux-gnu-g++ - export AR_x86_64_unknown_linux_gnu=x86_64-linux-gnu-ar - export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc fi elif [ "${TARGETPLATFORM}" = "linux/arm64" ]; then RUST_TARGET=aarch64-unknown-linux-gnu if [ "${TARGETPLATFORM}" != "${BUILDPLATFORM}" ]; then apt-get update && apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu - export CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc - export CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ - export AR_aarch64_unknown_linux_gnu=aarch64-linux-gnu-ar - export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc fi else echo "Unsupported target platform: ${TARGETPLATFORM}" From 12147f975440f772d278a067fcaadf5054b52288 Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Fri, 8 May 2026 15:52:10 -0700 Subject: [PATCH 04/13] c --- rust/otap-dataflow/cross-arch-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/otap-dataflow/cross-arch-build.sh b/rust/otap-dataflow/cross-arch-build.sh index 6043653ff0..b050682fa5 100755 --- a/rust/otap-dataflow/cross-arch-build.sh +++ b/rust/otap-dataflow/cross-arch-build.sh @@ -3,12 +3,12 @@ set -exu if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then RUST_TARGET=x86_64-unknown-linux-gnu if [ "${TARGETPLATFORM}" != "${BUILDPLATFORM}" ]; then - apt-get update && apt-get install -y gcc-x86-64-linux-gnu g++-x86-64-linux-gnu + apt-get update && apt-get install -y gcc-x86-64-linux-gnu fi elif [ "${TARGETPLATFORM}" = "linux/arm64" ]; then RUST_TARGET=aarch64-unknown-linux-gnu if [ "${TARGETPLATFORM}" != "${BUILDPLATFORM}" ]; then - apt-get update && apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu + apt-get update && apt-get install -y gcc-aarch64-linux-gnu fi else echo "Unsupported target platform: ${TARGETPLATFORM}" From 986bb406335375303b407d23f0ded14858aef60e Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Fri, 8 May 2026 15:57:34 -0700 Subject: [PATCH 05/13] c --- .github/workflows/dataflow-engine-binary-size.yml | 2 +- .../test_suites/integration/continuous/100klrps-docker.yaml | 6 +++--- .../integration/continuous/idle-state-template.yaml.j2 | 2 +- .../test_suites/integration/continuous/passthrough.yaml | 6 +++--- .../continuous/saturation-cores-template.yaml.j2 | 6 +++--- .../integration/nightly/100klrps-batch-sizes-docker.yaml | 6 +++--- .../integration/nightly/backpressure-docker.yaml | 6 +++--- .../integration/nightly/batch-processor-docker.yaml | 6 +++--- .../test_suites/integration/nightly/filter-docker.yaml | 6 +++--- .../test_suites/integration/nightly/otelcol-docker.yaml | 4 ++-- .../test_suites/integration/nightly/syslog-docker.yaml | 4 ++-- .../test_suites/integration/nightly/syslog-tcp-docker.yaml | 4 ++-- 12 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/dataflow-engine-binary-size.yml b/.github/workflows/dataflow-engine-binary-size.yml index 91d5e90e74..0caab899c2 100644 --- a/.github/workflows/dataflow-engine-binary-size.yml +++ b/.github/workflows/dataflow-engine-binary-size.yml @@ -53,7 +53,7 @@ jobs: - name: Get binary size and create report run: | - SIZE=$(docker run --rm --entrypoint stat df_engine:${{ matrix.platform.name }} -c %s /home/dataflow/df_engine) + SIZE=$(docker run --rm --entrypoint stat df_engine:${{ matrix.platform.name }} -c %s /home/nonroot/df_engine) SIZE_HR=$(numfmt --to=iec-i --suffix=B $SIZE) SIZE_MB=$(echo "scale=2; $SIZE / 1048576" | bc) echo "Binary size for ${{ matrix.platform.name }}: $SIZE_HR ($SIZE_MB MB)" diff --git a/tools/pipeline_perf_test/test_suites/integration/continuous/100klrps-docker.yaml b/tools/pipeline_perf_test/test_suites/integration/continuous/100klrps-docker.yaml index 911fa79dee..9e4557b67b 100644 --- a/tools/pipeline_perf_test/test_suites/integration/continuous/100klrps-docker.yaml +++ b/tools/pipeline_perf_test/test_suites/integration/continuous/100klrps-docker.yaml @@ -17,7 +17,7 @@ components: ### This doesn't work. # build-context: 'otel-arrow=../../' volumes: - - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" @@ -38,7 +38,7 @@ components: ports: - "8086:8080" volumes: - - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" @@ -59,7 +59,7 @@ components: ports: - "8087:8080" volumes: - - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" diff --git a/tools/pipeline_perf_test/test_suites/integration/continuous/idle-state-template.yaml.j2 b/tools/pipeline_perf_test/test_suites/integration/continuous/idle-state-template.yaml.j2 index c70201494c..0f53e14e35 100644 --- a/tools/pipeline_perf_test/test_suites/integration/continuous/idle-state-template.yaml.j2 +++ b/tools/pipeline_perf_test/test_suites/integration/continuous/idle-state-template.yaml.j2 @@ -10,7 +10,7 @@ components: ports: - "{{port}}:8080" volumes: - - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" diff --git a/tools/pipeline_perf_test/test_suites/integration/continuous/passthrough.yaml b/tools/pipeline_perf_test/test_suites/integration/continuous/passthrough.yaml index deb4812dfe..70aa287a66 100644 --- a/tools/pipeline_perf_test/test_suites/integration/continuous/passthrough.yaml +++ b/tools/pipeline_perf_test/test_suites/integration/continuous/passthrough.yaml @@ -9,7 +9,7 @@ components: ports: - "8085:8080" volumes: - - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" @@ -30,7 +30,7 @@ components: ports: - "8086:8080" volumes: - - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" @@ -51,7 +51,7 @@ components: ports: - "8087:8080" volumes: - - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" diff --git a/tools/pipeline_perf_test/test_suites/integration/continuous/saturation-cores-template.yaml.j2 b/tools/pipeline_perf_test/test_suites/integration/continuous/saturation-cores-template.yaml.j2 index c28a034031..f1dfbb154f 100644 --- a/tools/pipeline_perf_test/test_suites/integration/continuous/saturation-cores-template.yaml.j2 +++ b/tools/pipeline_perf_test/test_suites/integration/continuous/saturation-cores-template.yaml.j2 @@ -10,7 +10,7 @@ components: ports: - "8085:8080" volumes: - - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" @@ -31,7 +31,7 @@ components: ports: - "8086:8080" volumes: - - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" @@ -52,7 +52,7 @@ components: ports: - "8087:8080" volumes: - - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" diff --git a/tools/pipeline_perf_test/test_suites/integration/nightly/100klrps-batch-sizes-docker.yaml b/tools/pipeline_perf_test/test_suites/integration/nightly/100klrps-batch-sizes-docker.yaml index 64f1240de8..614e816f4e 100644 --- a/tools/pipeline_perf_test/test_suites/integration/nightly/100klrps-batch-sizes-docker.yaml +++ b/tools/pipeline_perf_test/test_suites/integration/nightly/100klrps-batch-sizes-docker.yaml @@ -8,7 +8,7 @@ components: ports: - "8085:8080" volumes: - - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" @@ -29,7 +29,7 @@ components: ports: - "8086:8080" volumes: - - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" @@ -50,7 +50,7 @@ components: ports: - "8087:8080" volumes: - - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" diff --git a/tools/pipeline_perf_test/test_suites/integration/nightly/backpressure-docker.yaml b/tools/pipeline_perf_test/test_suites/integration/nightly/backpressure-docker.yaml index 26036044e2..e5e981a28d 100644 --- a/tools/pipeline_perf_test/test_suites/integration/nightly/backpressure-docker.yaml +++ b/tools/pipeline_perf_test/test_suites/integration/nightly/backpressure-docker.yaml @@ -17,7 +17,7 @@ components: ### This doesn't work. # build-context: 'otel-arrow=../../' volumes: - - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" @@ -38,7 +38,7 @@ components: ports: - "8086:8080" volumes: - - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" @@ -59,7 +59,7 @@ components: ports: - "8087:8080" volumes: - - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" diff --git a/tools/pipeline_perf_test/test_suites/integration/nightly/batch-processor-docker.yaml b/tools/pipeline_perf_test/test_suites/integration/nightly/batch-processor-docker.yaml index 65c0b6a266..2ad901e416 100644 --- a/tools/pipeline_perf_test/test_suites/integration/nightly/batch-processor-docker.yaml +++ b/tools/pipeline_perf_test/test_suites/integration/nightly/batch-processor-docker.yaml @@ -8,7 +8,7 @@ components: ports: - "8085:8080" volumes: - - "test_suites/integration/configs/loadgen/config.rendered.yaml:/home/dataflow/config.yaml:ro" + - "test_suites/integration/configs/loadgen/config.rendered.yaml:/home/nonroot/config.yaml:ro" command: - "--config" - "./config.yaml" @@ -29,7 +29,7 @@ components: ports: - "8086:8080" volumes: - - "test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro" + - "test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro" command: - "--config" - "./config.yaml" @@ -50,7 +50,7 @@ components: ports: - "8087:8080" volumes: - - "test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro" + - "test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro" command: - "--config" - "./config.yaml" diff --git a/tools/pipeline_perf_test/test_suites/integration/nightly/filter-docker.yaml b/tools/pipeline_perf_test/test_suites/integration/nightly/filter-docker.yaml index 1b6289b4f0..54766d38d5 100644 --- a/tools/pipeline_perf_test/test_suites/integration/nightly/filter-docker.yaml +++ b/tools/pipeline_perf_test/test_suites/integration/nightly/filter-docker.yaml @@ -17,7 +17,7 @@ components: ### This doesn't work. # build-context: 'otel-arrow=../../' volumes: - - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" @@ -38,7 +38,7 @@ components: ports: - "8086:8080" volumes: - - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" @@ -59,7 +59,7 @@ components: ports: - "8087:8080" volumes: - - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" diff --git a/tools/pipeline_perf_test/test_suites/integration/nightly/otelcol-docker.yaml b/tools/pipeline_perf_test/test_suites/integration/nightly/otelcol-docker.yaml index 3428101b68..46af9755be 100644 --- a/tools/pipeline_perf_test/test_suites/integration/nightly/otelcol-docker.yaml +++ b/tools/pipeline_perf_test/test_suites/integration/nightly/otelcol-docker.yaml @@ -18,7 +18,7 @@ components: ### This doesn't work. # build-context: 'otel-arrow=../../' volumes: - - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" @@ -59,7 +59,7 @@ components: ports: - "8087:8080" volumes: - - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" diff --git a/tools/pipeline_perf_test/test_suites/integration/nightly/syslog-docker.yaml b/tools/pipeline_perf_test/test_suites/integration/nightly/syslog-docker.yaml index ceaf1d6a5a..6798104a87 100644 --- a/tools/pipeline_perf_test/test_suites/integration/nightly/syslog-docker.yaml +++ b/tools/pipeline_perf_test/test_suites/integration/nightly/syslog-docker.yaml @@ -33,7 +33,7 @@ components: ports: - "8086:8080" volumes: - - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" @@ -54,7 +54,7 @@ components: ports: - "8087:8080" volumes: - - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" diff --git a/tools/pipeline_perf_test/test_suites/integration/nightly/syslog-tcp-docker.yaml b/tools/pipeline_perf_test/test_suites/integration/nightly/syslog-tcp-docker.yaml index c5ea491cf8..4cbc15296f 100644 --- a/tools/pipeline_perf_test/test_suites/integration/nightly/syslog-tcp-docker.yaml +++ b/tools/pipeline_perf_test/test_suites/integration/nightly/syslog-tcp-docker.yaml @@ -33,7 +33,7 @@ components: ports: - "8086:8080" volumes: - - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" @@ -54,7 +54,7 @@ components: ports: - "8087:8080" volumes: - - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro' + - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro' command: - "--config" - "./config.yaml" From 357ab0cbb3926e83e055653ee31c18df2badf0ae Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Sun, 10 May 2026 13:58:39 -0700 Subject: [PATCH 06/13] c --- rust/otap-dataflow/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/otap-dataflow/Dockerfile b/rust/otap-dataflow/Dockerfile index c96772d665..36e0610ba5 100644 --- a/rust/otap-dataflow/Dockerfile +++ b/rust/otap-dataflow/Dockerfile @@ -7,7 +7,7 @@ # with the arrow enabled go collector build files in the repo root, the image # must be built with additional --build-context flags as follows: # docker build --build-context otel-arrow=../../ -f Dockerfile -t df_engine . -FROM --platform=$BUILDPLATFORM docker.io/rust:1.95@sha256:4a7e3a0c309c9bab658e469f842711bd595fae484936bc5d605e08ca0c631bf4 AS dataflow-build +FROM --platform=$BUILDPLATFORM docker.io/rust:1.95@sha256:a9cfb755b33f5bb872610cbdb25da61f527416b28fc9c052bbce4bef93e7799a AS dataflow-build WORKDIR /build/rust/dataflow ARG BUILDPLATFORM ARG TARGETPLATFORM From b9d4b140673bdb1c010c38c16c043f7d52490369 Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Sun, 10 May 2026 15:00:07 -0700 Subject: [PATCH 07/13] c --- rust/otap-dataflow/.cargo/config.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/rust/otap-dataflow/.cargo/config.toml b/rust/otap-dataflow/.cargo/config.toml index 03b6132779..f0559a4aaf 100644 --- a/rust/otap-dataflow/.cargo/config.toml +++ b/rust/otap-dataflow/.cargo/config.toml @@ -3,8 +3,6 @@ xtask = "run --package xtask --bin xtask --" [target.aarch64-unknown-linux-gnu] linker = "aarch64-linux-gnu-gcc" -rustflags = ["-C", "target-cpu=native"] [target.x86_64-unknown-linux-gnu] linker = "x86_64-linux-gnu-gcc" -rustflags = ["-C", "target-cpu=native"] \ No newline at end of file From 6d4981e0c9bf96a0f4ec256684fe3763d7d49ec4 Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Mon, 11 May 2026 13:02:57 -0700 Subject: [PATCH 08/13] Fix binary size tracking --- .github/workflows/dataflow-engine-binary-size.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dataflow-engine-binary-size.yml b/.github/workflows/dataflow-engine-binary-size.yml index 0caab899c2..85ed95800e 100644 --- a/.github/workflows/dataflow-engine-binary-size.yml +++ b/.github/workflows/dataflow-engine-binary-size.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: schedule: # Run daily at 00:00 UTC - - cron: '0 0 * * *' + - cron: "0 0 * * *" jobs: build-multiarch: @@ -53,11 +53,10 @@ jobs: - name: Get binary size and create report run: | - SIZE=$(docker run --rm --entrypoint stat df_engine:${{ matrix.platform.name }} -c %s /home/nonroot/df_engine) - SIZE_HR=$(numfmt --to=iec-i --suffix=B $SIZE) - SIZE_MB=$(echo "scale=2; $SIZE / 1048576" | bc) - echo "Binary size for ${{ matrix.platform.name }}: $SIZE_HR ($SIZE_MB MB)" - + cid=$(docker create df_engine:latest) + SIZE_MB=$(docker cp "$cid":/home/nonroot/df_engine - | wc -c | numfmt --to-unit=1048576 --format='%.2f') + docker container rm "$cid" + mkdir -p size-reports cat > size-reports/${{ matrix.platform.name }}.json << EOF [ From c8516aef26c3e9a2c18a871a676308f96566e959 Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Mon, 11 May 2026 13:04:36 -0700 Subject: [PATCH 09/13] Compute size for correct platform --- .github/workflows/dataflow-engine-binary-size.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dataflow-engine-binary-size.yml b/.github/workflows/dataflow-engine-binary-size.yml index 85ed95800e..54207f6f75 100644 --- a/.github/workflows/dataflow-engine-binary-size.yml +++ b/.github/workflows/dataflow-engine-binary-size.yml @@ -53,9 +53,10 @@ jobs: - name: Get binary size and create report run: | - cid=$(docker create df_engine:latest) + cid=$(docker create df_engine:${{ matrix.platform.name }}) SIZE_MB=$(docker cp "$cid":/home/nonroot/df_engine - | wc -c | numfmt --to-unit=1048576 --format='%.2f') docker container rm "$cid" + echo "Binary size for ${{ matrix.platform.name }}: $SIZE_HR ($SIZE_MB MB)" mkdir -p size-reports cat > size-reports/${{ matrix.platform.name }}.json << EOF From 9e48815ff4853749a5fd79ccc722b5abe8893666 Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Mon, 11 May 2026 13:05:41 -0700 Subject: [PATCH 10/13] Format without SIZE_HR --- .github/workflows/dataflow-engine-binary-size.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dataflow-engine-binary-size.yml b/.github/workflows/dataflow-engine-binary-size.yml index 54207f6f75..7bac931b70 100644 --- a/.github/workflows/dataflow-engine-binary-size.yml +++ b/.github/workflows/dataflow-engine-binary-size.yml @@ -56,7 +56,7 @@ jobs: cid=$(docker create df_engine:${{ matrix.platform.name }}) SIZE_MB=$(docker cp "$cid":/home/nonroot/df_engine - | wc -c | numfmt --to-unit=1048576 --format='%.2f') docker container rm "$cid" - echo "Binary size for ${{ matrix.platform.name }}: $SIZE_HR ($SIZE_MB MB)" + echo "Binary size for ${{ matrix.platform.name }}: $SIZE_MB MB" mkdir -p size-reports cat > size-reports/${{ matrix.platform.name }}.json << EOF From 0633249168cec2ee94942a02ac80ce59c1c1489a Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Mon, 11 May 2026 13:06:45 -0700 Subject: [PATCH 11/13] lint --- .github/workflows/dataflow-engine-binary-size.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dataflow-engine-binary-size.yml b/.github/workflows/dataflow-engine-binary-size.yml index 7bac931b70..af45bf7c19 100644 --- a/.github/workflows/dataflow-engine-binary-size.yml +++ b/.github/workflows/dataflow-engine-binary-size.yml @@ -57,7 +57,7 @@ jobs: SIZE_MB=$(docker cp "$cid":/home/nonroot/df_engine - | wc -c | numfmt --to-unit=1048576 --format='%.2f') docker container rm "$cid" echo "Binary size for ${{ matrix.platform.name }}: $SIZE_MB MB" - + mkdir -p size-reports cat > size-reports/${{ matrix.platform.name }}.json << EOF [ From 4b223d7a6c903ed5006b5fa0ec9c39429d678195 Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Mon, 11 May 2026 14:39:39 -0700 Subject: [PATCH 12/13] c --- rust/otap-dataflow/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/otap-dataflow/Dockerfile b/rust/otap-dataflow/Dockerfile index 36e0610ba5..674fe15b86 100644 --- a/rust/otap-dataflow/Dockerfile +++ b/rust/otap-dataflow/Dockerfile @@ -38,7 +38,7 @@ ENV FEATURES=${FEATURES} RUN ./cross-arch-build.sh # Runtime image -FROM gcr.io/distroless/cc-debian13:nonroot +FROM gcr.io/distroless/cc-debian13:nonroot@sha256:8f960b7fc6a5d6e28bb07f982655925d6206678bd9a6cde2ad00ddb5e2077d78 LABEL maintainer="The OpenTelemetry Authors" WORKDIR /home/nonroot From 17dcbafd3774c76e0e6ab5ffd439b413de210ceb Mon Sep 17 00:00:00 2001 From: Jake Dern Date: Mon, 18 May 2026 19:52:44 +0000 Subject: [PATCH 13/13] c --- rust/otap-dataflow/.cargo/config.toml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/rust/otap-dataflow/.cargo/config.toml b/rust/otap-dataflow/.cargo/config.toml index f0559a4aaf..86281707cc 100644 --- a/rust/otap-dataflow/.cargo/config.toml +++ b/rust/otap-dataflow/.cargo/config.toml @@ -1,8 +1,2 @@ [alias] xtask = "run --package xtask --bin xtask --" - -[target.aarch64-unknown-linux-gnu] -linker = "aarch64-linux-gnu-gcc" - -[target.x86_64-unknown-linux-gnu] -linker = "x86_64-linux-gnu-gcc"