diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index ce88fc7..df5cd41 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -25,7 +25,7 @@ jobs: name: End Status if: always() runs-on: ubuntu-latest - needs: [discover-modules, test-modules, test-scenarios, validate-structure, test-charts] + needs: [discover-modules, test-modules, test-scenarios, validate-structure, test-charts, test-merge] permissions: contents: read statuses: write @@ -83,7 +83,7 @@ jobs: - name: Install common dependencies run: | sudo apt-get update - sudo apt-get install -y jq curl wget + sudo apt-get install -y jq curl wget gawk - name: Set up Go (for jplot module) if: matrix.module == 'jplot' @@ -185,6 +185,141 @@ jobs: echo "✓ ${{ matrix.traffic }}/${{ matrix.scenario }} scenario executed successfully" + test-merge: + name: Test Multi-Pod Merge + runs-on: ubuntu-latest + needs: [test-modules, start-status] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y jq curl wget gawk + + - name: Install vegeta and jaggr + run: | + chmod +x modules/vegeta/install/install.sh + modules/vegeta/install/install.sh + + - name: Start test server + run: | + docker run -d \ + --name merge-test-server \ + -p 9998:9998 \ + -e PORT=9998 \ + ghcr.io/azure/aks-traffic-ingress-competitive-testing:8aba95806ff611e9939257e2c3c9f53b3af5f7a2 + + SERVER_READY=false + for i in {1..10}; do + if curl -s http://localhost:9998 > /dev/null; then + echo "Test server ready" + SERVER_READY=true + break + fi + echo "Waiting for server... (attempt $i/10)" + sleep 2 + done + + if [ "$SERVER_READY" != "true" ]; then + echo "ERROR: Test server failed to start after 10 attempts" + docker logs merge-test-server 2>&1 || true + exit 1 + fi + + - name: Run 4 simultaneous vegeta attacks + run: | + mkdir -p results/merge-test + + # Simulate 4 pods each attacking at 50 RPS for 15s (total: 200 RPS) + for pod in 0 1 2 3; do + echo "GET http://localhost:9998" | \ + vegeta attack -rate=50 -duration=15s -workers=2 \ + > "results/merge-test/pod${pod}.bin" & + done + + echo "Waiting for all 4 attacks to complete..." + wait + echo "All attacks finished" + + # Verify all .bin files are non-empty + for pod in 0 1 2 3; do + if [[ ! -s "results/merge-test/pod${pod}.bin" ]]; then + echo "ERROR: pod${pod}.bin is empty" + exit 1 + fi + echo "pod${pod}.bin: $(wc -c < results/merge-test/pod${pod}.bin) bytes" + done + + - name: Merge results and validate + run: | + chmod +x modules/vegeta/merge/merge.sh + + modules/vegeta/merge/merge.sh \ + --output-file results/merge-test/merged.json \ + results/merge-test/pod0.bin \ + results/merge-test/pod1.bin \ + results/merge-test/pod2.bin \ + results/merge-test/pod3.bin + + echo "=== Merged output ===" + cat results/merge-test/merged.json + echo "" + + # Verify the merged file is non-empty + if [[ ! -s results/merge-test/merged.json ]]; then + echo "ERROR: Merged output is empty" + exit 1 + fi + + # Verify per-second bucketing: 15s test should produce at least 12 lines + # (first and last second-buckets may be partial) + LINE_COUNT=$(wc -l < results/merge-test/merged.json) + echo "Merged output has ${LINE_COUNT} lines for a 15s test" + if [[ "${LINE_COUNT}" -lt 12 ]]; then + echo "ERROR: Expected at least 12 second-buckets for a 15s test, got ${LINE_COUNT}" + exit 1 + fi + + # Verify combined RPS is roughly 200 (4 pods x 50 RPS) + AVG_RPS=$(jq -r '.rps' results/merge-test/merged.json | awk '{s+=$1; n++} END {print int(s/n)}') + echo "Average merged RPS: ${AVG_RPS} (expected ~200)" + if [[ "${AVG_RPS}" -lt 120 ]]; then + echo "ERROR: Average RPS ${AVG_RPS} is too low (expected ~200 from 4x50 RPS)" + exit 1 + fi + + # Verify code histogram sums match reported rps for each line + while IFS= read -r line; do + RPS_VAL=$(echo "$line" | jq -r '.rps') + CODE_SUM=$(echo "$line" | jq -r '[.code.hist | to_entries[] | .value] | add // 0') + if [[ "${CODE_SUM}" -ne "${RPS_VAL}" ]]; then + echo "ERROR: Code histogram sum (${CODE_SUM}) does not match rps (${RPS_VAL})" + echo "Line: ${line}" + exit 1 + fi + done < results/merge-test/merged.json + + # Verify all lines have the expected JSON structure + while IFS= read -r line; do + for field in rps code.hist latency.p25 latency.p50 latency.p99 bytes_in.sum bytes_out.sum; do + if ! echo "$line" | jq -e ".${field}" > /dev/null 2>&1; then + echo "ERROR: Merged output line missing field '${field}'" + echo "Line: ${line}" + exit 1 + fi + done + done < results/merge-test/merged.json + + echo "✓ Multi-pod merge validation passed" + + - name: Cleanup + if: always() + run: | + docker stop merge-test-server > /dev/null 2>&1 || true + docker rm merge-test-server > /dev/null 2>&1 || true + validate-structure: name: Validate Project Structure runs-on: ubuntu-latest diff --git a/Dockerfile b/Dockerfile index 0c3b5e8..09bdf18 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,7 @@ RUN apt-get update && apt-get install -y \ wget \ git \ jq \ + gawk \ unzip \ bash \ ca-certificates \ @@ -103,6 +104,7 @@ RUN printf '%s\n' \ ' echo " install/ [args...] Run an install script (e.g. install/nginx)"' \ ' echo " setup/ [args...] Run a setup script (e.g. setup/ingress)"' \ ' echo " module// [args...] Run a module script (e.g. module/vegeta/run)"' \ + ' echo " merge [args...] Merge vegeta .bin files (modules/vegeta/merge/merge.sh)"' \ ' echo " server Start the HTTP server"' \ ' echo " bash -c \"...\" Run a custom command"' \ ' echo ""' \ @@ -171,6 +173,9 @@ RUN printf '%s\n' \ ' done' \ ' exit 1' \ ' fi' \ + 'elif [ "$1" = "merge" ]; then' \ + ' shift' \ + ' exec bash /app/modules/vegeta/merge/merge.sh "$@"' \ 'else' \ ' exec "$@"' \ 'fi' \ diff --git a/modules/vegeta/merge/merge.sh b/modules/vegeta/merge/merge.sh new file mode 100755 index 0000000..a3073af --- /dev/null +++ b/modules/vegeta/merge/merge.sh @@ -0,0 +1,165 @@ +#!/bin/bash + +# Script to merge multiple vegeta raw .bin files into jaggr-format per-second JSON output. +# Uses actual request timestamps for bucketing (not wall-clock time), so it works correctly +# on saved/replayed data and correctly interleaves results from pods that started at +# slightly different times. +# +# Note: The first and last second-buckets may be partial (vegeta doesn't start/stop exactly +# on second boundaries). Consumers should account for this when validating RPS values. + +set -eo pipefail + +show_usage() { + echo "Usage: $0 [--output-file FILE] [bin_file2 ...]" + echo "" + echo "Merges one or more raw vegeta .bin files into jaggr-format per-second JSON output." + echo "" + echo "Options:" + echo " --output-file FILE Write output to FILE (default: stdout)" + echo " -h, --help Show this help message" + echo "" + echo "Examples:" + echo " $0 results/pod0.bin results/pod1.bin" + echo " $0 --output-file merged.json results/pod0.bin results/pod1.bin results/pod2.bin" + echo " $0 results/single.bin" +} + +output_file="" +bin_files=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --output-file) + output_file="$2" + shift 2 + ;; + -h|--help) + show_usage + exit 0 + ;; + -*) + echo "Error: Unknown option: $1" + show_usage + exit 1 + ;; + *) + bin_files+=("$1") + shift + ;; + esac +done + +if [[ ${#bin_files[@]} -eq 0 ]]; then + echo "Error: At least one .bin file is required" + show_usage + exit 1 +fi + +# Validate all input files exist and are non-empty +for f in "${bin_files[@]}"; do + if [[ ! -f "$f" ]]; then + echo "Error: File not found: $f" + exit 1 + fi + if [[ ! -s "$f" ]]; then + echo "Error: File is empty: $f" + exit 1 + fi +done + +echo "Merging ${#bin_files[@]} .bin file(s)..." >&2 + +# Pipeline: +# 1. vegeta encode --to csv on all input files (vegeta round-robins through them) +# 2. Sort by timestamp column (column 1, nanoseconds since epoch) +# 3. gawk to bucket by second and compute per-bucket aggregates +# 4. Output one JSON line per second-bucket +merge_results() { + vegeta encode --to csv "${bin_files[@]}" | \ + sort -t, -k1,1n | \ + gawk -F, ' + function floor_val(x) { + return int(x) + } + function flush_bucket() { + if (bucket_count == 0) return + + # Compute latency percentiles + asort(latencies, sorted_lat) + n = bucket_count + p25_idx = floor_val(n * 0.25) + if (p25_idx < 1) p25_idx = 1 + p50_idx = floor_val(n * 0.50) + if (p50_idx < 1) p50_idx = 1 + p99_idx = floor_val(n * 0.99) + if (p99_idx < 1) p99_idx = 1 + + p25_val = sorted_lat[p25_idx] + p50_val = sorted_lat[p50_idx] + p99_val = sorted_lat[p99_idx] + + # Build code histogram JSON + code_hist = "" + for (code in code_counts) { + if (code_hist != "") code_hist = code_hist "," + code_hist = code_hist "\"" code "\":" code_counts[code] + } + + printf "{\"rps\":%d,\"code\":{\"hist\":{%s}},\"latency\":{\"p25\":%d,\"p50\":%d,\"p99\":%d},\"bytes_in\":{\"sum\":%d},\"bytes_out\":{\"sum\":%d}}\n", \ + bucket_count, code_hist, p25_val, p50_val, p99_val, bytes_in_sum, bytes_out_sum + } + + BEGIN { + current_second = -1 + bucket_count = 0 + bytes_in_sum = 0 + bytes_out_sum = 0 + } + + { + # CSV columns: timestamp_ns, status_code, latency_ns, bytes_out, bytes_in, error + timestamp_ns = $1 + status_code = $2 + latency_ns = $3 + bytes_out = $4 + bytes_in = $5 + + # Bucket by second (integer division of nanoseconds by 1e9) + this_second = floor_val(timestamp_ns / 1000000000) + + if (current_second == -1) { + current_second = this_second + } + + if (this_second != current_second) { + flush_bucket() + + # Reset for new bucket + current_second = this_second + bucket_count = 0 + bytes_in_sum = 0 + bytes_out_sum = 0 + delete code_counts + delete latencies + } + + bucket_count++ + latencies[bucket_count] = latency_ns + 0 + code_counts[status_code] += 1 + bytes_in_sum += bytes_in + 0 + bytes_out_sum += bytes_out + 0 + } + + END { + flush_bucket() + } + ' +} + +if [[ -n "$output_file" ]]; then + merge_results > "$output_file" + echo "Merged output written to ${output_file}" >&2 +else + merge_results +fi diff --git a/modules/vegeta/run/run.sh b/modules/vegeta/run/run.sh index e2a3f8b..4cacbcb 100755 --- a/modules/vegeta/run/run.sh +++ b/modules/vegeta/run/run.sh @@ -3,10 +3,11 @@ # Script to run Vegeta HTTP load testing attacks # https://github.com/tsenart/vegeta -set -ex +set -exo pipefail filepath=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) statefile="${filepath}/../statefile.json" +binfile="${filepath}/../statefile.bin" show_usage() { echo "Usage: $0 [OPTIONS]" @@ -98,27 +99,14 @@ run_vegeta_attack() { echo "No additional headers provided." fi - # Run attack and generate report - # Use a temp file to stream results to disk first, avoiding O(n) memory usage - # This prevents OOM at high RPS (e.g., 20k RPS for 3 minutes = 3.6M requests) - local temp_results - temp_results=$(mktemp) - - echo "Writing vegeta attack output to temp file: $temp_results" + # Run attack with streaming tee pipeline: + # - tee saves raw binary results to .bin file (kept for downstream merge) + # - vegeta encode | jaggr processes results in real-time (correct per-second bucketing) + echo "Streaming vegeta attack to ${binfile} and jaggr..." echo "GET $target_url" | \ - "${attack_cmd[@]}" > "$temp_results" - - # Verify attack produced output - if [[ ! -s "$temp_results" ]]; then - echo "ERROR: Vegeta attack produced no output" - rm -f "$temp_results" - return 1 - fi - - echo "Processing results from disk ($(wc -c < "$temp_results") bytes)..." - # Read line-by-line and add delay at end to let jaggr complete its time bucket - # jaggr aggregates by wall-clock time, so we need to wait for the bucket to flush - (vegeta encode "$temp_results" | while IFS= read -r line; do echo "$line"; done; sleep 2) | \ + "${attack_cmd[@]}" | \ + tee "$binfile" | \ + vegeta encode | \ jaggr @count=rps \ hist\[100,200,300,400,500\]:code \ p25,p50,p99:latency \ @@ -126,7 +114,19 @@ run_vegeta_attack() { sum:bytes_out | \ tee "$statefile" - rm -f "$temp_results" + # Verify the .bin file is non-empty + if [[ ! -s "$binfile" ]]; then + echo "ERROR: Vegeta attack produced no output (${binfile} is empty)" + return 1 + fi + + # Verify the statefile is non-empty + if [[ ! -s "$statefile" ]]; then + echo "ERROR: jaggr produced no output (${statefile} is empty)" + return 1 + fi + + echo "Raw binary results saved to ${binfile} ($(wc -c < "$binfile") bytes)" } # If script is run directly (not sourced) diff --git a/modules/vegeta/test/test.sh b/modules/vegeta/test/test.sh index 529f2f8..27e9cdd 100755 --- a/modules/vegeta/test/test.sh +++ b/modules/vegeta/test/test.sh @@ -29,6 +29,8 @@ cleanup() { # Clean up any test state files rm -f "${MODULE_DIR}/statefile.json" || true + rm -f "${MODULE_DIR}/statefile.bin" || true + rm -f /tmp/vegeta-test-*.bin /tmp/vegeta-test-*.csv /tmp/vegeta-test-merge-*.json /tmp/vegeta-test-synthetic* || true } # Set trap to cleanup on exit @@ -104,8 +106,8 @@ chmod +x "${MODULE_DIR}/run/run.sh" cd "${PROJECT_ROOT}" "${MODULE_DIR}/run/run.sh" \ --target-url "http://localhost:${TEST_PORT}" \ - --rate 5 \ - --duration 2s \ + --rate 50 \ + --duration 10s \ --workers 2 # Verify statefile was created @@ -132,16 +134,18 @@ if ! grep -q "rps" "${STATEFILE}"; then exit 1 fi -# Check that we have successful HTTP 200 responses -STATUS_200_COUNT=$(head -n 1 "${STATEFILE}" | jq -r '.code.hist["200"] // 0' 2>/dev/null || echo "0") -if [[ "${STATUS_200_COUNT}" =~ ^[0-9]+$ ]] && [[ "${STATUS_200_COUNT}" -gt 0 ]]; then - echo "✓ Found ${STATUS_200_COUNT} successful HTTP 200 responses" -else - echo "ERROR: Expected HTTP 200 responses but found: ${STATUS_200_COUNT}" - echo "First line of state file:" - head -n 1 "${STATEFILE}" 2>/dev/null || echo "State file not found or empty" - exit 1 -fi +# Check that the sum of status codes in the histogram equals the reported rps +# (validates every request was accounted for, regardless of status code) +while IFS= read -r line; do + RPS_VAL=$(echo "$line" | jq -r '.rps') + CODE_SUM=$(echo "$line" | jq -r '[.code.hist | to_entries[] | .value] | add // 0') + if [[ "${CODE_SUM}" -ne "${RPS_VAL}" ]]; then + echo "ERROR: Code histogram sum (${CODE_SUM}) does not match rps (${RPS_VAL})" + echo "Line: ${line}" + exit 1 + fi +done < "${STATEFILE}" +echo "✓ All status code histogram sums match reported rps" echo "✓ State file content test passed" @@ -173,8 +177,8 @@ echo "8. Testing different attack parameters..." # Test with different parameters "${MODULE_DIR}/run/run.sh" \ --target-url "http://localhost:${TEST_PORT}" \ - --rate 10 \ - --duration 1s \ + --rate 500 \ + --duration 20s \ --workers 1 echo "✓ Parameter variation test passed" @@ -182,8 +186,8 @@ echo "✓ Parameter variation test passed" echo "9. Testing header-only named invocation..." HEADER_ONLY_OUTPUT=$("${MODULE_DIR}/run/run.sh" \ --target-url "http://localhost:${TEST_PORT}" \ - --rate 10 \ - --duration 1s \ + --rate 500 \ + --duration 20s \ --request-headers "X-Test-Header: header-only" 2>&1) echo "Header-only output:" printf '%s\n' "${HEADER_ONLY_OUTPUT}" @@ -207,8 +211,8 @@ echo "✓ Header-only invocation test passed" echo "10. Testing workers plus headers named invocation..." WORKERS_AND_HEADERS_OUTPUT=$("${MODULE_DIR}/run/run.sh" \ --target-url "http://localhost:${TEST_PORT}" \ - --rate 10 \ - --duration 1s \ + --rate 500 \ + --duration 20s \ --workers 1 \ --request-headers "X-Test-Header: with-workers" 2>&1) echo "Workers and headers output:" @@ -253,5 +257,385 @@ fi echo "✓ Positional invocation failure test passed" +echo "12. Testing raw .bin file is produced alongside statefile..." +BINFILE="${MODULE_DIR}/statefile.bin" +if [[ ! -f "${BINFILE}" ]]; then + echo "ERROR: Binary file was not created at ${BINFILE}" + exit 1 +fi + +if [[ ! -s "${BINFILE}" ]]; then + echo "ERROR: Binary file is empty" + exit 1 +fi + +# Verify it's valid vegeta binary by running vegeta encode on it +if ! vegeta encode "${BINFILE}" | head -n 1 | jq . > /dev/null 2>&1; then + echo "ERROR: Binary file is not valid vegeta binary format" + exit 1 +fi + +echo "✓ Raw .bin file test passed" + +echo "13. Testing per-second bucketing is correct..." +# Run a fresh attack with known rate and duration +rm -f "${STATEFILE}" "${BINFILE}" +"${MODULE_DIR}/run/run.sh" \ + --target-url "http://localhost:${TEST_PORT}" \ + --rate 50 \ + --duration 10s \ + --workers 2 + +# Count lines in statefile — a 10s test should produce ~10 lines (one per second) +# Allow some slack for edge effects on the first/last partial second +LINE_COUNT=$(wc -l < "${STATEFILE}") +echo "Statefile has ${LINE_COUNT} lines (second-buckets) for a 10s test" +if [[ "${LINE_COUNT}" -lt 8 ]]; then + echo "ERROR: Expected at least 8 second-buckets for a 10s test, got ${LINE_COUNT}" + echo "Statefile content:" + cat "${STATEFILE}" + exit 1 +fi + +# Check that each line's rps value is reasonable (roughly near 50, not collapsed or empty) +MIN_REASONABLE_RPS=38 # 0.75x the target rate +MAX_REASONABLE_RPS=63 # 1.25x the target rate +while IFS= read -r line; do + RPS_VAL=$(echo "$line" | jq -r '.rps // 0') + if [[ "${RPS_VAL}" -gt "${MAX_REASONABLE_RPS}" ]]; then + echo "ERROR: RPS value ${RPS_VAL} is unreasonably high (expected near 50, max ${MAX_REASONABLE_RPS})" + echo "This suggests results were collapsed instead of bucketed per-second" + echo "Line: ${line}" + exit 1 + fi + if [[ "${RPS_VAL}" -lt "${MIN_REASONABLE_RPS}" ]]; then + echo "ERROR: RPS value ${RPS_VAL} is unreasonably low (expected near 50, min ${MIN_REASONABLE_RPS})" + echo "Line: ${line}" + exit 1 + fi +done < "${STATEFILE}" + +echo "✓ Per-second bucketing test passed" + +echo "14. Testing merge.sh with a single .bin file..." +chmod +x "${MODULE_DIR}/merge/merge.sh" + +# Run a vegeta attack to produce a .bin file +rm -f "${STATEFILE}" "${BINFILE}" +"${MODULE_DIR}/run/run.sh" \ + --target-url "http://localhost:${TEST_PORT}" \ + --rate 50 \ + --duration 10s \ + --workers 2 + +MERGE_OUTPUT="/tmp/vegeta-test-merge-single.json" +rm -f "${MERGE_OUTPUT}" + +"${MODULE_DIR}/merge/merge.sh" --output-file "${MERGE_OUTPUT}" "${BINFILE}" + +# Validate output file exists and is non-empty +if [[ ! -s "${MERGE_OUTPUT}" ]]; then + echo "ERROR: Merge output file is empty or missing" + exit 1 +fi + +# Validate each line is valid JSON with expected fields +while IFS= read -r line; do + for field in rps "code" "latency" "bytes_in" "bytes_out"; do + if ! echo "$line" | jq -e ".${field}" > /dev/null 2>&1; then + echo "ERROR: Merge output line missing expected field '${field}'" + echo "Line: ${line}" + exit 1 + fi + done + # Check latency subfields + for subfield in p25 p50 p99; do + if ! echo "$line" | jq -e ".latency.${subfield}" > /dev/null 2>&1; then + echo "ERROR: Merge output line missing latency.${subfield}" + echo "Line: ${line}" + exit 1 + fi + done +done < "${MERGE_OUTPUT}" + +# Verify line count roughly matches the 10s test duration +# The first and last second-buckets may be partial, so expect ~10 lines from a 10s test +MERGE_LINE_COUNT=$(wc -l < "${MERGE_OUTPUT}") +echo "Merge output has ${MERGE_LINE_COUNT} lines (second-buckets) for a 10s test" +if [[ "${MERGE_LINE_COUNT}" -lt 8 ]]; then + echo "ERROR: Expected at least 8 second-buckets for a 10s test, got ${MERGE_LINE_COUNT}" + cat "${MERGE_OUTPUT}" + exit 1 +fi + +# Verify each line's code histogram sum matches rps +# Skip first and last lines when checking RPS bounds (they may be partial edge buckets) +MERGE_TOTAL_LINES=$(wc -l < "${MERGE_OUTPUT}") +MERGE_LINE_NUM=0 +MIN_REASONABLE_RPS=38 # 0.75x the target rate +MAX_REASONABLE_RPS=63 # 1.25x the target rate +while IFS= read -r line; do + MERGE_LINE_NUM=$((MERGE_LINE_NUM + 1)) + RPS_VAL=$(echo "$line" | jq -r '.rps // 0') + CODE_SUM=$(echo "$line" | jq -r '[.code.hist | to_entries[] | .value] | add // 0') + if [[ "${CODE_SUM}" -ne "${RPS_VAL}" ]]; then + echo "ERROR: Merge code histogram sum (${CODE_SUM}) does not match rps (${RPS_VAL})" + echo "Line: ${line}" + exit 1 + fi + # Skip RPS bounds check on first and last lines (partial edge buckets) + if [[ "${MERGE_LINE_NUM}" -eq 1 ]] || [[ "${MERGE_LINE_NUM}" -eq "${MERGE_TOTAL_LINES}" ]]; then + continue + fi + if [[ "${RPS_VAL}" -gt "${MAX_REASONABLE_RPS}" ]]; then + echo "ERROR: Merge RPS value ${RPS_VAL} is unreasonably high (expected near 50, max ${MAX_REASONABLE_RPS})" + echo "This suggests results were collapsed instead of bucketed per-second" + echo "Line: ${line}" + exit 1 + fi + if [[ "${RPS_VAL}" -lt "${MIN_REASONABLE_RPS}" ]]; then + echo "ERROR: Merge RPS value ${RPS_VAL} is unreasonably low (expected near 50, min ${MIN_REASONABLE_RPS})" + echo "Line: ${line}" + exit 1 + fi +done < "${MERGE_OUTPUT}" + +echo "✓ Merge single .bin file test passed" + +echo "15. Testing merge.sh combining multiple simultaneous .bin files..." +# Run two vegeta attacks in parallel so their timestamps actually overlap +BIN_FILE_1="/tmp/vegeta-test-attack1.bin" +BIN_FILE_2="/tmp/vegeta-test-attack2.bin" +rm -f "${BIN_FILE_1}" "${BIN_FILE_2}" + +# Launch both attacks simultaneously in background +echo "GET http://localhost:${TEST_PORT}" | \ + vegeta attack -rate=50 -duration=10s -workers=2 > "${BIN_FILE_1}" & +PID1=$! + +echo "GET http://localhost:${TEST_PORT}" | \ + vegeta attack -rate=50 -duration=10s -workers=2 > "${BIN_FILE_2}" & +PID2=$! + +# Wait for both to finish +wait "$PID1" "$PID2" + +# Verify both produced output +if [[ ! -s "${BIN_FILE_1}" ]]; then + echo "ERROR: Attack 1 produced no output" + exit 1 +fi +if [[ ! -s "${BIN_FILE_2}" ]]; then + echo "ERROR: Attack 2 produced no output" + exit 1 +fi + +# Run merge.sh on both files without --output-file, capture stdout +MULTI_MERGE_OUTPUT=$("${MODULE_DIR}/merge/merge.sh" "${BIN_FILE_1}" "${BIN_FILE_2}") + +# Verify line count roughly matches the 10s test duration +# The first and last second-buckets may be partial, so expect ~10 lines from a 10s test +MULTI_MERGE_LINE_COUNT=$(echo "$MULTI_MERGE_OUTPUT" | wc -l) +echo "Multi-file merge output has ${MULTI_MERGE_LINE_COUNT} lines (second-buckets) for a 10s test" +if [[ "${MULTI_MERGE_LINE_COUNT}" -lt 8 ]]; then + echo "ERROR: Expected at least 8 second-buckets for a 10s test, got ${MULTI_MERGE_LINE_COUNT}" + echo "$MULTI_MERGE_OUTPUT" + exit 1 +fi + +# Verify each line's code histogram sum matches rps +# Skip first and last lines when checking RPS bounds (they may be partial edge buckets) +MULTI_TOTAL_LINES=$(echo "$MULTI_MERGE_OUTPUT" | wc -l) +MULTI_LINE_NUM=0 +MIN_REASONABLE_RPS=75 # 0.75x the combined target rate of ~100 +MAX_REASONABLE_RPS=125 # 1.25x the combined target rate of ~100 +while IFS= read -r line; do + MULTI_LINE_NUM=$((MULTI_LINE_NUM + 1)) + RPS_VAL=$(echo "$line" | jq -r '.rps // 0') + CODE_SUM=$(echo "$line" | jq -r '[.code.hist | to_entries[] | .value] | add // 0') + if [[ "${CODE_SUM}" -ne "${RPS_VAL}" ]]; then + echo "ERROR: Multi-merge code histogram sum (${CODE_SUM}) does not match rps (${RPS_VAL})" + echo "Line: ${line}" + exit 1 + fi + # Skip RPS bounds check on first and last lines (partial edge buckets) + if [[ "${MULTI_LINE_NUM}" -eq 1 ]] || [[ "${MULTI_LINE_NUM}" -eq "${MULTI_TOTAL_LINES}" ]]; then + continue + fi + if [[ "${RPS_VAL}" -gt "${MAX_REASONABLE_RPS}" ]]; then + echo "ERROR: Multi-merge RPS value ${RPS_VAL} is unreasonably high (expected near 100, max ${MAX_REASONABLE_RPS})" + echo "This suggests results were collapsed instead of bucketed per-second" + echo "Line: ${line}" + exit 1 + fi + if [[ "${RPS_VAL}" -lt "${MIN_REASONABLE_RPS}" ]]; then + echo "ERROR: Multi-merge RPS value ${RPS_VAL} is unreasonably low (expected near 100, min ${MIN_REASONABLE_RPS})" + echo "Line: ${line}" + exit 1 + fi +done <<< "$MULTI_MERGE_OUTPUT" + +echo "✓ Merge multiple simultaneous .bin files test passed" + +echo "16. Testing merge.sh latency percentile logic with multi-pod synthetic data..." +# Simulate two pods attacking the same server by creating two separate .bin files +# with overlapping timestamps in the same second-buckets. This verifies that merge.sh +# correctly interleaves requests from multiple sources and computes accurate percentiles +# across the combined data. +# +# Strategy: Each pod produces 3 second-buckets (edge, interior, edge). +# The interior bucket (second 1) will contain the combined requests from both pods. +# The edge buckets (seconds 0 and 2) have fewer requests and are expected to be partial. +# +# Pod A interior bucket: 50 requests with latencies 1ms, 3ms, 5ms, ..., 99ms (odd ms) +# Pod B interior bucket: 50 requests with latencies 2ms, 4ms, 6ms, ..., 100ms (even ms) +# Combined: 100 requests with latencies 1ms, 2ms, 3ms, ..., 100ms +# +# Expected percentiles for the interior bucket: +# p25 = sorted[int(100 * 0.25)] = sorted[25] = 25ms = 25000000ns +# p50 = sorted[int(100 * 0.50)] = sorted[50] = 50ms = 50000000ns +# p99 = sorted[int(100 * 0.99)] = sorted[99] = 99ms = 99000000ns + +SYNTHETIC_CSV_A="/tmp/vegeta-test-synthetic-a.csv" +SYNTHETIC_CSV_B="/tmp/vegeta-test-synthetic-b.csv" +SYNTHETIC_BIN_A="/tmp/vegeta-test-synthetic-a.bin" +SYNTHETIC_BIN_B="/tmp/vegeta-test-synthetic-b.bin" +SYNTHETIC_OUT="/tmp/vegeta-test-synthetic-merged.json" +rm -f "${SYNTHETIC_CSV_A}" "${SYNTHETIC_CSV_B}" "${SYNTHETIC_BIN_A}" "${SYNTHETIC_BIN_B}" "${SYNTHETIC_OUT}" + +# CSV columns: timestamp_ns, status_code, latency_ns, bytes_out, bytes_in, error, +# body(base64), attack_name, seq, method, url, headers(base64) +BASE_TS=1700000000000000000 + +# --- Pod A: odd-millisecond latencies --- +SEQ=0 + +# Bucket 0 (edge): 10 padding requests +for i in $(seq 1 10); do + SEQ=$((SEQ + 1)) + TS=$((BASE_TS + i * 1000000)) + echo "${TS},200,5000000,0,13,,,,${SEQ},GET,http://localhost/," +done > "${SYNTHETIC_CSV_A}" + +# Bucket 1 (interior): 50 requests with latencies 1ms, 3ms, 5ms, ..., 99ms +for i in $(seq 1 50); do + SEQ=$((SEQ + 1)) + TS=$((BASE_TS + 1000000000 + i * 1000000)) + LATENCY=$(( (2 * i - 1) * 1000000 )) # 1ms, 3ms, 5ms, ..., 99ms + echo "${TS},200,${LATENCY},0,13,,,,${SEQ},GET,http://localhost/," +done >> "${SYNTHETIC_CSV_A}" + +# Bucket 2 (edge): 10 padding requests +for i in $(seq 1 10); do + SEQ=$((SEQ + 1)) + TS=$((BASE_TS + 2000000000 + i * 1000000)) + echo "${TS},200,5000000,0,13,,,,${SEQ},GET,http://localhost/," +done >> "${SYNTHETIC_CSV_A}" + +# --- Pod B: even-millisecond latencies --- +SEQ=0 + +# Bucket 0 (edge): 10 padding requests +for i in $(seq 1 10); do + SEQ=$((SEQ + 1)) + TS=$((BASE_TS + (i + 10) * 1000000)) # offset slightly so timestamps don't collide + echo "${TS},200,5000000,0,13,,,,${SEQ},GET,http://localhost/," +done > "${SYNTHETIC_CSV_B}" + +# Bucket 1 (interior): 50 requests with latencies 2ms, 4ms, 6ms, ..., 100ms +for i in $(seq 1 50); do + SEQ=$((SEQ + 1)) + TS=$((BASE_TS + 1000000000 + (i + 50) * 1000000)) # offset within same second + LATENCY=$(( 2 * i * 1000000 )) # 2ms, 4ms, 6ms, ..., 100ms + echo "${TS},200,${LATENCY},0,13,,,,${SEQ},GET,http://localhost/," +done >> "${SYNTHETIC_CSV_B}" + +# Bucket 2 (edge): 10 padding requests +for i in $(seq 1 10); do + SEQ=$((SEQ + 1)) + TS=$((BASE_TS + 2000000000 + (i + 10) * 1000000)) + echo "${TS},200,5000000,0,13,,,,${SEQ},GET,http://localhost/," +done >> "${SYNTHETIC_CSV_B}" + +# Convert both CSVs to vegeta binary format +vegeta encode --to gob < "${SYNTHETIC_CSV_A}" > "${SYNTHETIC_BIN_A}" +vegeta encode --to gob < "${SYNTHETIC_CSV_B}" > "${SYNTHETIC_BIN_B}" + +if [[ ! -s "${SYNTHETIC_BIN_A}" ]] || [[ ! -s "${SYNTHETIC_BIN_B}" ]]; then + echo "ERROR: Failed to create synthetic .bin files" + exit 1 +fi + +# Merge both pod files — this is the actual multi-pod merge scenario +"${MODULE_DIR}/merge/merge.sh" --output-file "${SYNTHETIC_OUT}" "${SYNTHETIC_BIN_A}" "${SYNTHETIC_BIN_B}" + +if [[ ! -s "${SYNTHETIC_OUT}" ]]; then + echo "ERROR: Merge of synthetic multi-pod data produced no output" + exit 1 +fi + +# Should be exactly 3 lines (edge bucket, interior bucket, edge bucket) +SYNTH_LINES=$(wc -l < "${SYNTHETIC_OUT}") +if [[ "${SYNTH_LINES}" -ne 3 ]]; then + echo "ERROR: Expected 3 lines from synthetic multi-pod data, got ${SYNTH_LINES}" + cat "${SYNTHETIC_OUT}" + exit 1 +fi + +# Validate the interior bucket (line 2) which has the known combined data +SYNTH_LINE=$(sed -n '2p' "${SYNTHETIC_OUT}") + +# Verify rps = 100 (50 from pod A + 50 from pod B) +SYNTH_RPS=$(echo "$SYNTH_LINE" | jq -r '.rps') +if [[ "${SYNTH_RPS}" -ne 100 ]]; then + echo "ERROR: Expected rps=100 (50+50), got ${SYNTH_RPS}" + echo "$SYNTH_LINE" + exit 1 +fi + +# Verify all 100 responses are code 200 +SYNTH_200=$(echo "$SYNTH_LINE" | jq -r '.code.hist["200"]') +if [[ "${SYNTH_200}" -ne 100 ]]; then + echo "ERROR: Expected code 200 count=100, got ${SYNTH_200}" + echo "$SYNTH_LINE" + exit 1 +fi + +# Verify latency percentiles across the combined (interleaved) data +# Combined sorted latencies: 1ms, 2ms, 3ms, ..., 100ms +# p25 = sorted[25] = 25ms = 25000000ns +SYNTH_P25=$(echo "$SYNTH_LINE" | jq -r '.latency.p25') +if [[ "${SYNTH_P25}" -ne 25000000 ]]; then + echo "ERROR: Expected latency.p25=25000000, got ${SYNTH_P25}" + echo "$SYNTH_LINE" + exit 1 +fi + +# p50 = sorted[50] = 50ms = 50000000ns +SYNTH_P50=$(echo "$SYNTH_LINE" | jq -r '.latency.p50') +if [[ "${SYNTH_P50}" -ne 50000000 ]]; then + echo "ERROR: Expected latency.p50=50000000, got ${SYNTH_P50}" + echo "$SYNTH_LINE" + exit 1 +fi + +# p99 = sorted[99] = 99ms = 99000000ns +SYNTH_P99=$(echo "$SYNTH_LINE" | jq -r '.latency.p99') +if [[ "${SYNTH_P99}" -ne 99000000 ]]; then + echo "ERROR: Expected latency.p99=99000000, got ${SYNTH_P99}" + echo "$SYNTH_LINE" + exit 1 +fi + +# Verify bytes_in.sum = 13 * 100 = 1300 (50 from each pod, 13 bytes each) +SYNTH_BYTES_IN=$(echo "$SYNTH_LINE" | jq -r '.bytes_in.sum') +if [[ "${SYNTH_BYTES_IN}" -ne 1300 ]]; then + echo "ERROR: Expected bytes_in.sum=1300, got ${SYNTH_BYTES_IN}" + echo "$SYNTH_LINE" + exit 1 +fi + +echo "✓ Multi-pod merge latency percentile logic test passed" + echo "" -echo "🎉 All Vegeta module tests passed!" +echo "All Vegeta module tests passed!"