ci: add weaver live-check job for host_metrics_receiver

.github/workflows/rust-ci.yml

@@ -707,6 +707,172 @@ jobs:
             emitted_phase1_metric_shapes_match_weaver_semconv --lib -- --ignored
         working-directory: ./rust/otap-dataflow
 
+  # Alternative semconv drift check: runs the host_metrics_receiver end-to-end
+  # against an OTLP `weaver registry live-check` listener and fails if Weaver
+  # reports any `violation` finding. This validates the real OTLP wire output
+  # rather than the in-process projection.
+  #
+  # Intentionally NOT in `rust-required-status-check.needs` for now: it runs in
+  # parallel with `host-metrics-semconv` so the two approaches can be compared
+  # before deciding which one to keep.
+  host-metrics-weaver-live-check:
+    runs-on: ubuntu-latest
+    env:
+      WEAVER_VERSION: v0.23.0
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          submodules: true
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          repository: open-telemetry/semantic-conventions
+          # Pinned to the same version the receiver targets
+          # (rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/semconv.rs).
+          ref: v1.41.0
+          path: semantic-conventions
+      - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9
+        with:
+          toolchain: stable
+      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
+        with:
+          workspaces: ./rust/otap-dataflow
+          cache-bin: false
+      - name: Install Weaver
+        run: |
+          set -euo pipefail
+          mkdir -p "$HOME/.weaver/bin"
+          curl -sSL -o /tmp/weaver.tar.xz \
+            "https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/weaver-x86_64-unknown-linux-gnu.tar.xz"
+          tar -xf /tmp/weaver.tar.xz -C "$HOME/.weaver/bin" --strip-components=1
+          echo "$HOME/.weaver/bin" >> "$GITHUB_PATH"
+      - name: Verify Weaver
+        run: weaver --version
+      - name: Build df_engine
+        run: cargo build --release --bin df_engine
+        working-directory: ./rust/otap-dataflow
+      - name: Run host_metrics_receiver against weaver live-check
+        working-directory: ${{ github.workspace }}
+        run: |
+          set -euo pipefail
+          mkdir -p live-check-report
+
+          echo "::group::Start weaver live-check listener"
+          weaver registry live-check \
+            --registry semantic-conventions/model \
+            --input-source otlp \
+            --otlp-grpc-port 4317 \
+            --admin-port 4320 \
+            --inactivity-timeout 60 \
+            --format json \
+            --output live-check-report \
+            > live-check-report/weaver.log 2>&1 &
+          WEAVER_PID=$!
+          echo "weaver pid: $WEAVER_PID"
+          echo "::endgroup::"
+
+          # Wait for the OTLP listener to come up.
+          for i in $(seq 1 30); do
+            if (echo > /dev/tcp/127.0.0.1/4317) >/dev/null 2>&1; then
+              break
+            fi
+            sleep 1
+          done
+
+          echo "::group::Start df_engine"
+          ./rust/otap-dataflow/target/release/df_engine \
+            --config rust/otap-dataflow/configs/host-metrics-otlp.yaml \
+            --num-cores 1 \
+            > live-check-report/df_engine.log 2>&1 &
+          ENGINE_PID=$!
+          echo "df_engine pid: $ENGINE_PID"
+          echo "::endgroup::"
+
+          # Allow several scrape iterations (collection_interval = 2s in the
+          # config), then enough time for the OTLP exporter to flush.
+          sleep 12
+
+          echo "::group::Stop services"
+          kill -INT "$ENGINE_PID" || true
+          # Best effort: give df_engine a moment to flush before tearing down.
+          sleep 2
+          # Tell weaver to stop accepting and finalize the report.
+          curl -fsS -X POST "http://127.0.0.1:4320/stop" || true
+          wait "$ENGINE_PID" 2>/dev/null || true
+          set +e
+          wait "$WEAVER_PID"
+          WEAVER_EXIT=$?
+          set -e
+          echo "weaver exit code: $WEAVER_EXIT"
+          echo "::endgroup::"
+
+          echo "::group::Weaver log (tail)"
+          tail -n 200 live-check-report/weaver.log || true
+          echo "::endgroup::"
+
+          echo "::group::df_engine log (tail)"
+          tail -n 100 live-check-report/df_engine.log || true
+          echo "::endgroup::"
+
+          exit $WEAVER_EXIT
+      - name: Summarize live-check findings
+        if: always()
+        working-directory: ${{ github.workspace }}
+        run: |
+          set -euo pipefail
+          REPORT=live-check-report/live_check.json
+          if [ ! -s "$REPORT" ]; then
+            echo "No live-check report found at $REPORT" | tee -a "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+
+          python3 - "$REPORT" <<'PY' | tee -a "$GITHUB_STEP_SUMMARY"
+          import json, sys
+          d = json.load(open(sys.argv[1]))
+          s = d.get("statistics", {}) or {}
+          levels = s.get("advice_level_counts", {}) or {}
+          violations = levels.get("violation", 0)
+          improvements = levels.get("improvement", 0)
+          informations = levels.get("information", 0)
+          status_emoji = "❌" if violations else "✅"
+
+          print(f"## {status_emoji} Weaver live-check (host_metrics_receiver)")
+          print()
+          print(f"- Samples received: **{len(d.get('samples', []))}**")
+          by_type = s.get("total_entities_by_type", {}) or {}
+          if by_type:
+              parts = ", ".join(f"{v} {k}" for k, v in sorted(by_type.items()))
+              print(f"- Entities checked: **{s.get('total_entities', 0)}** ({parts})")
+          print()
+          print("| Level | Count |")
+          print("| --- | ---: |")
+          print(f"| violation | {violations} |")
+          print(f"| improvement | {improvements} |")
+          print(f"| information | {informations} |")
+          print()
+
+          types = s.get("advice_type_counts", {}) or {}
+          if types:
+              print("### Findings by type")
+              print()
+              print("| Count | Type |")
+              print("| ---: | --- |")
+              for t, c in sorted(types.items(), key=lambda x: -x[1]):
+                  print(f"| {c} | {t} |")
+              print()
+
+          if violations:
+              print("> Violations were reported. See artifact `weaver-live-check-host-metrics` for the full JSON report.")
+          else:
+              print("> No semantic-convention violations. Improvements/informations are advisory.")
+          PY
+      - name: Upload live-check report
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: weaver-live-check-host-metrics
+          path: live-check-report
+          if-no-files-found: warn
+
   # Required matrix combinations for deny: otap-dataflow only
   deny_required:
     runs-on: ubuntu-latest

rust/otap-dataflow/configs/host-metrics-otlp.yaml

@@ -0,0 +1,44 @@
+version: otel_dataflow/v1
+engine: { }
+groups:
+  default:
+    pipelines:
+      main:
+        # host_metrics_receiver -> OTLP/gRPC exporter pipeline.
+        #
+        # Used by the `host-metrics-weaver-live-check` CI job to feed real
+        # OTLP traffic from the host metrics receiver into a `weaver registry
+        # live-check` listener. The downstream listener then asserts that
+        # what's emitted matches the OpenTelemetry semantic conventions.
+        #
+        # The host_metrics_receiver requires a single-core source pipeline,
+        # so this config is meant to be run as:
+        #
+        #   df_engine --config configs/host-metrics-otlp.yaml --num-cores 1
+        #
+        # Linux-only. The downstream OTLP/gRPC endpoint defaults to the
+        # Weaver live-check default port (4317).
+
+        policies:
+          channel_capacity:
+            control:
+              node: 100
+              pipeline: 100
+            pdata: 128
+
+        nodes:
+          receiver:
+            type: urn:otel:receiver:host_metrics
+            config:
+              # Short interval so a brief CI run produces several scrapes.
+              collection_interval: 2s
+              initial_delay: 0s
+
+          exporter:
+            type: exporter:otlp_grpc
+            config:
+              grpc_endpoint: "http://127.0.0.1:4317"
+
+        connections:
+          - from: receiver
+            to: exporter