feat: Replace static musl build on alpine with glibc based build on gcr distroless

.github/workflows/dataflow-engine-binary-size.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
   schedule:
     # Run daily at 00:00 UTC
-    - cron: '0 0 * * *'
+    - cron: "0 0 * * *"
 
 jobs:
   build-multiarch:
@@ -53,10 +53,10 @@ jobs:
 
       - name: Get binary size and create report
         run: |
-          SIZE=$(docker run --rm --entrypoint stat df_engine:${{ matrix.platform.name }} -c %s /home/dataflow/df_engine)
-          SIZE_HR=$(numfmt --to=iec-i --suffix=B $SIZE)
-          SIZE_MB=$(echo "scale=2; $SIZE / 1048576" | bc)
-          echo "Binary size for ${{ matrix.platform.name }}: $SIZE_HR ($SIZE_MB MB)"
+          cid=$(docker create df_engine:${{ matrix.platform.name }})
+          SIZE_MB=$(docker cp "$cid":/home/nonroot/df_engine - | wc -c | numfmt --to-unit=1048576 --format='%.2f')
+          docker container rm "$cid"
+          echo "Binary size for ${{ matrix.platform.name }}: $SIZE_MB MB"
 
           mkdir -p size-reports
           cat > size-reports/${{ matrix.platform.name }}.json << EOF

rust/otap-dataflow/.cargo/config.toml

@@ -1,10 +1,8 @@
 [alias]
 xtask = "run --package xtask --bin xtask --"
 
-[target.aarch64-unknown-linux-musl]
+[target.aarch64-unknown-linux-gnu]
 linker = "aarch64-linux-gnu-gcc"
-rustflags = ["-C", "target-cpu=native"]
 
-[target.x86_64-unknown-linux-musl]
+[target.x86_64-unknown-linux-gnu]
 linker = "x86_64-linux-gnu-gcc"
-rustflags = ["-C", "target-cpu=native"]

rust/otap-dataflow/Dockerfile

@@ -37,21 +37,11 @@ ENV FEATURES=${FEATURES}
 # Build dataflow engine
 RUN ./cross-arch-build.sh
 
-# The runtime image
-FROM docker.io/alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11
+# Runtime image
+FROM gcr.io/distroless/cc-debian13:nonroot@sha256:8f960b7fc6a5d6e28bb07f982655925d6206678bd9a6cde2ad00ddb5e2077d78
 LABEL maintainer="The OpenTelemetry Authors"
 
-# Passing --build-arg BUILD=1 adds additional debugging features to the image.
-ARG DEBUG="0"
-RUN if [ "$DEBUG" = "1" ]; then apk add --no-cache gdb; fi
-
-RUN addgroup dataflow \
-  && adduser \
-  --ingroup dataflow \
-  --disabled-password \
-  dataflow
-WORKDIR /home/dataflow
-
-COPY --from=dataflow-build --chown=dataflow:dataflow /build/rust/dataflow/df_engine .
-USER dataflow
-ENTRYPOINT ["/home/dataflow/df_engine"]
+WORKDIR /home/nonroot
+COPY --from=dataflow-build --chown=nonroot:nonroot /build/rust/dataflow/df_engine .
+USER nonroot
+ENTRYPOINT ["/home/nonroot/df_engine"]

rust/otap-dataflow/cross-arch-build.sh

@@ -1,56 +1,14 @@
 set -exu
 
 if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then
-  RUST_TARGET=x86_64-unknown-linux-musl
-  apt-get update && apt-get install -y musl-tools
-  export CC_x86_64_unknown_linux_musl=musl-gcc
+  RUST_TARGET=x86_64-unknown-linux-gnu
   if [ "${TARGETPLATFORM}" != "${BUILDPLATFORM}" ]; then
     apt-get update && apt-get install -y gcc-x86-64-linux-gnu
   fi
 elif [ "${TARGETPLATFORM}" = "linux/arm64" ]; then
-  RUST_TARGET=aarch64-unknown-linux-musl
-  if [ "${BUILDPLATFORM}" = "linux/arm64" ]; then
-    # Native arm64 builder: use the musl.cc arm64-hosted native toolchain.
-    # musl-tools/musl-gcc is avoided because its unprefixed compiler name causes
-    # jemalloc's configure to misdetect the build as cross-compilation
-    # (build=aarch64-linux-gnu vs host=aarch64-linux-musl), which breaks atomic
-    # detection. The musl.cc native toolchain provides a properly-prefixed
-    # aarch64-linux-musl-gcc that runs natively on arm64 with musl headers.
-    MUSL_NATIVE_TARBALL=aarch64-linux-musl-native.tgz
-    MUSL_NATIVE_SHA256=daf336cafa2c3c7daf42f6a46edc960f10a181fcf15ab9f1c43b192e8ad2a069
-    MUSL_NATIVE_DIR=/opt/aarch64-linux-musl-native
-    curl -sSfL "https://musl.cc/${MUSL_NATIVE_TARBALL}" -o "/tmp/${MUSL_NATIVE_TARBALL}"
-    echo "${MUSL_NATIVE_SHA256}  /tmp/${MUSL_NATIVE_TARBALL}" | sha256sum -c -
-    tar xz -C /opt -f "/tmp/${MUSL_NATIVE_TARBALL}"
-    rm "/tmp/${MUSL_NATIVE_TARBALL}"
-    # The native tarball ships aarch64-linux-musl-gcc-ar but not aarch64-linux-musl-ar.
-    # cmake-based crates (e.g. zstd-sys) look for the latter by name, so symlink it.
-    ln -s "${MUSL_NATIVE_DIR}/bin/aarch64-linux-musl-gcc-ar" "${MUSL_NATIVE_DIR}/bin/aarch64-linux-musl-ar"
-    export CC_aarch64_unknown_linux_musl=${MUSL_NATIVE_DIR}/bin/aarch64-linux-musl-gcc
-    export CXX_aarch64_unknown_linux_musl=${MUSL_NATIVE_DIR}/bin/aarch64-linux-musl-g++
-    export AR_aarch64_unknown_linux_musl=${MUSL_NATIVE_DIR}/bin/aarch64-linux-musl-ar
-    export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=${MUSL_NATIVE_DIR}/bin/aarch64-linux-musl-gcc
-  elif [ "${BUILDPLATFORM}" = "linux/amd64" ]; then
-    # Cross-compile (amd64 -> arm64): Debian's gcc-aarch64-linux-gnu uses glibc
-    # system headers; glibc >= 2.38 maps _GNU_SOURCE to _ISOC23_SOURCE, which
-    # redirects strtol/sscanf to __isoc23_strtol/__isoc23_sscanf. Those C23
-    # symbols don't exist in musl libc, causing link failures in C dependencies
-    # that define _GNU_SOURCE (e.g. aws-lc-sys/bcm.c). Use the musl.cc
-    # x86_64-hosted cross-toolchain which ships its own musl headers instead.
-    MUSL_CROSS_TARBALL=aarch64-linux-musl-cross.tgz
-    MUSL_CROSS_SHA256=c909817856d6ceda86aa510894fa3527eac7989f0ef6e87b5721c58737a06c38
-    MUSL_CROSS_DIR=/opt/aarch64-linux-musl-cross
-    curl -sSfL "https://musl.cc/${MUSL_CROSS_TARBALL}" -o "/tmp/${MUSL_CROSS_TARBALL}"
-    echo "${MUSL_CROSS_SHA256}  /tmp/${MUSL_CROSS_TARBALL}" | sha256sum -c -
-    tar xz -C /opt -f "/tmp/${MUSL_CROSS_TARBALL}"
-    rm "/tmp/${MUSL_CROSS_TARBALL}"
-    export CC_aarch64_unknown_linux_musl=${MUSL_CROSS_DIR}/bin/aarch64-linux-musl-gcc
-    export CXX_aarch64_unknown_linux_musl=${MUSL_CROSS_DIR}/bin/aarch64-linux-musl-g++
-    export AR_aarch64_unknown_linux_musl=${MUSL_CROSS_DIR}/bin/aarch64-linux-musl-ar
-    export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=${MUSL_CROSS_DIR}/bin/aarch64-linux-musl-gcc
-  else
-    echo "Unsupported build platform for arm64 target: ${BUILDPLATFORM}"
-    exit 1
+  RUST_TARGET=aarch64-unknown-linux-gnu
+  if [ "${TARGETPLATFORM}" != "${BUILDPLATFORM}" ]; then
+    apt-get update && apt-get install -y gcc-aarch64-linux-gnu
   fi
 else
   echo "Unsupported target platform: ${TARGETPLATFORM}"

tools/pipeline_perf_test/test_suites/integration/continuous/100klrps-docker.yaml

@@ -17,7 +17,7 @@ components:
         ###   This doesn't work.
         #     build-context: 'otel-arrow=../../'
         volumes:
-          - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"
@@ -38,7 +38,7 @@ components:
         ports:
           - "8086:8080"
         volumes:
-          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"
@@ -59,7 +59,7 @@ components:
         ports:
           - "8087:8080"
         volumes:
-          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"

tools/pipeline_perf_test/test_suites/integration/continuous/idle-state-template.yaml.j2

@@ -10,7 +10,7 @@ components:
         ports:
           - "{{port}}:8080"
         volumes:
-          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"

tools/pipeline_perf_test/test_suites/integration/continuous/passthrough.yaml

@@ -9,7 +9,7 @@ components:
         ports:
           - "8085:8080"
         volumes:
-          - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"
@@ -30,7 +30,7 @@ components:
         ports:
           - "8086:8080"
         volumes:
-          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"
@@ -51,7 +51,7 @@ components:
         ports:
           - "8087:8080"
         volumes:
-          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"

tools/pipeline_perf_test/test_suites/integration/continuous/saturation-cores-template.yaml.j2

@@ -10,7 +10,7 @@ components:
         ports:
           - "8085:8080"
         volumes:
-          - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"
@@ -31,7 +31,7 @@ components:
         ports:
           - "8086:8080"
         volumes:
-          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"
@@ -52,7 +52,7 @@ components:
         ports:
           - "8087:8080"
         volumes:
-          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"

tools/pipeline_perf_test/test_suites/integration/nightly/100klrps-batch-sizes-docker.yaml

@@ -8,7 +8,7 @@ components:
         ports:
           - "8085:8080"
         volumes:
-          - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"
@@ -29,7 +29,7 @@ components:
         ports:
           - "8086:8080"
         volumes:
-          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"
@@ -50,7 +50,7 @@ components:
         ports:
           - "8087:8080"
         volumes:
-          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"

tools/pipeline_perf_test/test_suites/integration/nightly/backpressure-docker.yaml

@@ -17,7 +17,7 @@ components:
         ###   This doesn't work.
         #     build-context: 'otel-arrow=../../'
         volumes:
-          - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"
@@ -38,7 +38,7 @@ components:
         ports:
           - "8086:8080"
         volumes:
-          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"
@@ -59,7 +59,7 @@ components:
         ports:
           - "8087:8080"
         volumes:
-          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"

tools/pipeline_perf_test/test_suites/integration/nightly/batch-processor-docker.yaml

@@ -8,7 +8,7 @@ components:
         ports:
           - "8085:8080"
         volumes:
-          - "test_suites/integration/configs/loadgen/config.rendered.yaml:/home/dataflow/config.yaml:ro"
+          - "test_suites/integration/configs/loadgen/config.rendered.yaml:/home/nonroot/config.yaml:ro"
         command:
           - "--config"
           - "./config.yaml"
@@ -29,7 +29,7 @@ components:
         ports:
           - "8086:8080"
         volumes:
-          - "test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro"
+          - "test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro"
         command:
           - "--config"
           - "./config.yaml"
@@ -50,7 +50,7 @@ components:
         ports:
           - "8087:8080"
         volumes:
-          - "test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro"
+          - "test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro"
         command:
           - "--config"
           - "./config.yaml"

tools/pipeline_perf_test/test_suites/integration/nightly/filter-docker.yaml

@@ -17,7 +17,7 @@ components:
         ###   This doesn't work.
         #     build-context: 'otel-arrow=../../'
         volumes:
-          - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"
@@ -38,7 +38,7 @@ components:
         ports:
           - "8086:8080"
         volumes:
-          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"
@@ -59,7 +59,7 @@ components:
         ports:
           - "8087:8080"
         volumes:
-          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"

tools/pipeline_perf_test/test_suites/integration/nightly/otelcol-docker.yaml

@@ -18,7 +18,7 @@ components:
         ###   This doesn't work.
         #     build-context: 'otel-arrow=../../'
         volumes:
-          - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/loadgen/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"
@@ -59,7 +59,7 @@ components:
         ports:
           - "8087:8080"
         volumes:
-          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"

tools/pipeline_perf_test/test_suites/integration/nightly/syslog-docker.yaml

@@ -33,7 +33,7 @@ components:
         ports:
           - "8086:8080"
         volumes:
-          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"
@@ -54,7 +54,7 @@ components:
         ports:
           - "8087:8080"
         volumes:
-          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"

tools/pipeline_perf_test/test_suites/integration/nightly/syslog-tcp-docker.yaml

@@ -33,7 +33,7 @@ components:
         ports:
           - "8086:8080"
         volumes:
-          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"
@@ -54,7 +54,7 @@ components:
         ports:
           - "8087:8080"
         volumes:
-          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+          - 'test_suites/integration/configs/backend/config.rendered.yaml:/home/nonroot/config.yaml:ro'
         command:
           - "--config"
           - "./config.yaml"