Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion .github/workflows/build-and-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ on:
transaction_exclusion_api_changed:
required: true
type: string
maru_changed:
required: true
type: string
build_linea_besu_package:
required: true
type: string
Expand All @@ -57,6 +60,9 @@ on:
transaction_exclusion_api_image_tagged:
required: true
type: string
maru_image_tagged:
required: true
type: string
outputs:
linea_besu_package_tag:
value: ${{ jobs.linea_besu_package_build_and_upload.outputs.linea_besu_package_tag }}
Expand Down Expand Up @@ -127,6 +133,16 @@ jobs:
push_image: ${{ inputs.push_image }}
secrets: inherit

maru:
uses: ./.github/workflows/maru-build-and-publish.yml
if: ${{ always() && (inputs.maru_changed == 'true' || (inputs.push_image && inputs.maru_image_tagged != 'true')) }}
with:
commit_tag: ${{ inputs.commit_tag }}
develop_tag: ${{ inputs.develop_tag }}
image_name: consensys/maru
push_image: ${{ inputs.push_image }}
secrets: inherit

linea_besu_package_build_and_upload:
uses: ./.github/workflows/linea-besu-package-build-and-upload.yml
if: ${{ always() && inputs.build_linea_besu_package == 'true' && !inputs.push_image }}
Expand All @@ -138,7 +154,7 @@ jobs:
# If all jobs are skipped, the workflow will still succeed.
always_succeed:
runs-on: ubuntu-24.04
needs: [coordinator, postman, prover, transaction_exclusion_api, native_yield_automation_service, linea_besu_package_build_and_upload]
needs: [coordinator, postman, prover, transaction_exclusion_api, native_yield_automation_service, maru, linea_besu_package_build_and_upload]
if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'success') && !contains(needs.*.result, 'cancelled') }}
steps:
- name: Ensure Workflow Success
Expand Down
18 changes: 18 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ jobs:
lido-governance-monitor: ${{ steps.filter.outputs.lido-governance-monitor }}
tracer: ${{ steps.filter.outputs.tracer }}
tracer-constraints: ${{ steps.filter.outputs.tracer-constraints }}
maru: ${{ steps.filter.outputs.maru }}
has-changes-requiring-build: ${{ steps.exclusion-filter.outputs.has-changes-requiring-build }}
has-changes-requiring-linea-besu-package-build: ${{ steps.filter.outputs.linea-sequencer-plugin == 'true' || steps.filter.outputs.tracer == 'true' || steps.filter.outputs.tracer-constraints == 'true' || steps.filter.outputs.linea-besu == 'true' || steps.filter.outputs.linea-besu-package == 'true' }}
steps:
Expand Down Expand Up @@ -160,6 +161,17 @@ jobs:
- 'tracer-constraints/**'
- '.github/workflows/tracer-constraints-*.yml'
- 'tracer/gradle/corset.gradle'
maru:
- 'maru/**'
- 'buildSrc/**'
- 'jvm-libs/**'
- 'gradle/**'
- 'build.gradle'
- 'gradle.properties'
- 'settings.gradle'
- '.github/workflows/maru-*.yml'
- '.github/workflows/build-and-publish.yml'
- '.github/workflows/main.yml'

- name: Filter out commit changes
uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 #v3.0.2
Expand Down Expand Up @@ -191,6 +203,7 @@ jobs:
transaction_exclusion_api_changed: ${{ needs.filter-commit-changes.outputs.transaction-exclusion-api }}
native_yield_automation_service_changed: ${{ needs.filter-commit-changes.outputs.native-yield-automation-service }}
lido_governance_monitor_changed: ${{ needs.filter-commit-changes.outputs.lido-governance-monitor }}
maru_changed: ${{ needs.filter-commit-changes.outputs.maru }}
secrets: inherit

testing:
Expand All @@ -211,6 +224,7 @@ jobs:
lido_governance_monitor_changed: ${{ needs.filter-commit-changes.outputs.lido-governance-monitor }}
tracer_changed: ${{ needs.filter-commit-changes.outputs.tracer }}
constraints_changed: ${{ needs.filter-commit-changes.outputs.tracer-constraints }}
maru_changed: ${{ needs.filter-commit-changes.outputs.maru }}
secrets: inherit

get-has-changes-requiring-e2e-testing:
Expand Down Expand Up @@ -247,13 +261,15 @@ jobs:
transaction_exclusion_api_changed: ${{ needs.filter-commit-changes.outputs.transaction-exclusion-api }}
native_yield_automation_service_changed: ${{ needs.filter-commit-changes.outputs.native-yield-automation-service }}
lido_governance_monitor_changed: ${{ needs.filter-commit-changes.outputs.lido-governance-monitor }}
maru_changed: ${{ needs.filter-commit-changes.outputs.maru }}
build_linea_besu_package: ${{ needs.filter-commit-changes.outputs.has-changes-requiring-linea-besu-package-build }}
coordinator_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_coordinator }}
postman_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_postman }}
prover_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_prover }}
transaction_exclusion_api_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_transaction_exclusion_api }}
native_yield_automation_service_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_native_yield_automation_service }}
lido_governance_monitor_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_lido_governance_monitor }}
maru_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_maru }}
secrets: inherit

run-e2e-tests:
Expand Down Expand Up @@ -302,13 +318,15 @@ jobs:
transaction_exclusion_api_changed: ${{ needs.filter-commit-changes.outputs.transaction-exclusion-api }}
native_yield_automation_service_changed: ${{ needs.filter-commit-changes.outputs.native-yield-automation-service }}
lido_governance_monitor_changed: ${{ needs.filter-commit-changes.outputs.lido-governance-monitor }}
maru_changed: ${{ needs.filter-commit-changes.outputs.maru }}
build_linea_besu_package: ${{ needs.filter-commit-changes.outputs.has-changes-requiring-linea-besu-package-build }}
coordinator_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_coordinator }}
postman_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_postman }}
prover_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_prover }}
transaction_exclusion_api_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_transaction_exclusion_api }}
native_yield_automation_service_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_native_yield_automation_service }}
lido_governance_monitor_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_lido_governance_monitor }}
maru_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_maru }}
secrets: inherit

cleanup-deployments:
Expand Down
125 changes: 125 additions & 0 deletions .github/workflows/maru-build-and-publish.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
name: maru-build-and-publish

permissions:
contents: read
actions: read
packages: write

on:
workflow_call:
inputs:
commit_tag:
required: false
type: string
default: ''
develop_tag:
required: true
type: string
image_name:
required: true
type: string
push_image:
required: false
type: boolean
default: false
upload_dist_artifact: # Used for release process to reuse the artifact
required: false
type: boolean
default: false
outputs:
commit_tag:
value: ${{ jobs.build-and-publish.outputs.commit_tag }}
secrets:
DOCKERHUB_USERNAME:
required: false
DOCKERHUB_TOKEN:
required: false
workflow_dispatch:
inputs:
commit_tag:
description: 'Image tag, if not given, HEAD commit hash will be used'
required: false
type: string
default: ''
develop_tag:
description: 'Image tag will be "develop" if target branch is main'
required: true
type: choice
options:
- develop
default: 'develop'
image_name:
description: 'Image name'
required: true
type: string
default: 'consensys/maru'
push_image:
description: 'Toggle whether to push image to docker registry'
required: false
type: boolean
default: true

concurrency:
group: maru-docker-build-and-publish-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
build-and-publish:
runs-on: gha-runner-scale-set-ubuntu-24-amd64-large
name: Maru build and publish
env:
COMMIT_TAG: ${{ inputs.commit_tag }}
DEVELOP_TAG: ${{ inputs.develop_tag }}
IMAGE_NAME: ${{ inputs.image_name }}
PUSH_IMAGE: ${{ inputs.push_image }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
outputs:
commit_tag: ${{ env.COMMIT_TAG }}
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Set commit tag if not given
if: ${{ inputs.commit_tag == '' }}
run: |
# For PR, GITHUB_SHA is NOT the last commit pushed onto the PR branch - https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#pull_request
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
COMMIT_TAG=$(git rev-parse --short ${{ github.event.pull_request.head.sha }})
else
COMMIT_TAG=$(git rev-parse --short $GITHUB_SHA)
fi
echo "COMMIT_TAG=$COMMIT_TAG" >> $GITHUB_ENV
echo "COMMIT_TAG=$COMMIT_TAG"
- name: Login to Docker Hub
if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Setup Java and Gradle
uses: ./.github/actions/setup-java-and-gradle
- name: Build dist
run: |
./gradlew :maru:app:installDist
- name: Upload distribution artifact
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need to upload it?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just aligned it with what we have for coordinator. However this can be removed and integrate with the new release process. This can be done in a follow up PR.

if: ${{ inputs.upload_dist_artifact }}
uses: actions/upload-artifact@v4
with:
name: maru-distribution
path: maru/app/build/install/app/
retention-days: 1
- name: Build and publish Docker image
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we should instead integrate it with the new release process

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just aligned it with what we have for coordinator. However this can be removed and integrate with the new release process. This can be done in a follow up PR.

uses: ./.github/actions/docker-build-publish
with:
docker_context: maru/app
image_name: ${{ env.IMAGE_NAME }}
image_tag: ${{ env.COMMIT_TAG }}
develop_tag: ${{ env.DEVELOP_TAG }}
push_image: ${{ env.PUSH_IMAGE }}
dockerfile_path: maru/app/Dockerfile
requires_qemu: ${{ env.PUSH_IMAGE }}
build_contexts: |
libs=./maru/app/build/install/app/lib/
maru=./maru/app/build/libs/
save_artifact: ${{ env.PUSH_IMAGE == 'false' }}
artifact_name: linea-maru
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Chaos Testing
name: maru-chaos-testing

on:
workflow_dispatch:
Expand Down Expand Up @@ -31,6 +31,11 @@ on:
required: false
type: boolean
default: false
secrets:
DOCKERHUB_USERNAME:
required: false
DOCKERHUB_TOKEN:
required: false

permissions: {} # lock everything by default (least-privilege)

Expand All @@ -45,8 +50,8 @@ jobs:
if: ${{ inputs.consensus-client-changed == 'true' }}
env:
COMMIT_TAG: ${{ inputs.commit_tag }}
DOCKER_ORG_NAME: ${{ secrets.DOCKER_ORG_NAME }}
DOCKER_REPO_TOKEN: ${{ secrets.DOCKER_REPO_TOKEN }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
runs-on: [ gha-runner-scale-set-ubuntu-24-amd64-large ]
name: chaos tests
# useful for debugging flaky tests.
Expand All @@ -57,17 +62,13 @@ jobs:
- name: Checkout
uses: actions/checkout@v6
- name: Login to Docker Hub
if: ${{ env.DOCKER_ORG_NAME != '' && env.DOCKER_REPO_TOKEN != '' }}
uses: docker/login-action@v3
if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
username: ${{ secrets.DOCKER_ORG_NAME }}
password: ${{ secrets.DOCKER_REPO_TOKEN }}
- uses: actions/setup-java@v4
with:
distribution: temurin
java-version: 25
- name: Setup Gradle
uses: gradle/actions/setup-gradle@8379f6a1328ee0e06e2bb424dadb7b159856a326 # 4.4.0
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Setup Java and Gradle
uses: ./.github/actions/setup-java-and-gradle
- name: Install helm
uses: azure/setup-helm@v4.3.1
with:
Expand All @@ -85,8 +86,8 @@ jobs:
- name: Run chaos tests
id: chaos-tests
timeout-minutes: 30
working-directory: maru/chaos-testing
run: |
cd chaos-testing
mkdir -p ~/.kube
export KUBECONFIG=~/.kube/k3s-server
make k3s-reload # installs k3s K8S implementation
Expand All @@ -99,38 +100,38 @@ jobs:
if: failure()
run: |
export KUBECONFIG=~/.kube/k3s-server
mkdir -p chaos-testing/tmp/pod-logs
mkdir -p maru/chaos-testing/tmp/pod-logs
echo "Collecting logs from all pods..."
for pod in $(kubectl get pods -n default -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do
echo "Collecting logs for $pod"
kubectl logs "$pod" -n default --tail=5000 > "chaos-testing/tmp/pod-logs/${pod}.log" 2>&1 || true
kubectl logs "$pod" -n default --tail=5000 > "maru/chaos-testing/tmp/pod-logs/${pod}.log" 2>&1 || true
# Also collect previous container logs (if pod restarted)
kubectl logs "$pod" -n default --previous --tail=5000 > "chaos-testing/tmp/pod-logs/${pod}-previous.log" 2>&1 || true
kubectl logs "$pod" -n default --previous --tail=5000 > "maru/chaos-testing/tmp/pod-logs/${pod}-previous.log" 2>&1 || true
done
# Collect pod status/describe for context
kubectl get pods -n default -o wide > "chaos-testing/tmp/pod-logs/pod-status.txt" 2>&1 || true
kubectl describe pods -n default > "chaos-testing/tmp/pod-logs/pod-describe.txt" 2>&1 || true
kubectl get pods -n default -o wide > "maru/chaos-testing/tmp/pod-logs/pod-status.txt" 2>&1 || true
kubectl describe pods -n default > "maru/chaos-testing/tmp/pod-logs/pod-describe.txt" 2>&1 || true
echo "Log collection complete"
ls -la chaos-testing/tmp/pod-logs/
ls -la maru/chaos-testing/tmp/pod-logs/

- name: Collect port-forward logs
if: failure()
run: |
echo "Collecting port-forward logs..."
if [ -d chaos-testing/tmp/pf ]; then
ls -la chaos-testing/tmp/pf/
if [ -d maru/chaos-testing/tmp/pf ]; then
ls -la maru/chaos-testing/tmp/pf/
else
echo "No port-forward log directory found"
fi
# Also collect port-forward summary files
for f in chaos-testing/tmp/port-forward-*.txt; do
for f in maru/chaos-testing/tmp/port-forward-*.txt; do
if [ -f "$f" ]; then
echo "=== $f ==="
cat "$f"
fi
done
# Check for any surviving port-forward processes
ps -o pid=,command= -ax | grep 'kubectl port-forward' | grep -v grep > chaos-testing/tmp/pod-logs/port-forward-processes.txt 2>&1 || true
ps -o pid=,command= -ax | grep 'kubectl port-forward' | grep -v grep > maru/chaos-testing/tmp/pod-logs/port-forward-processes.txt 2>&1 || true
echo "Port-forward process collection complete"

- name: Store reports and pod logs
Expand All @@ -139,15 +140,15 @@ jobs:
with:
name: chaos-test-reports-and-logs
path: |
**/build/reports/tests/
chaos-testing/tmp/pod-logs/
chaos-testing/tmp/pf/
chaos-testing/tmp/port-forward-*.txt
maru/**/build/reports/tests/
maru/chaos-testing/tmp/pod-logs/
maru/chaos-testing/tmp/pf/
maru/chaos-testing/tmp/port-forward-*.txt

# SSH debugging session - on failure (if requested)
- name: Setup upterm session after failure
if: ${{ (failure() || steps.chaos-tests.outcome == 'failure') && inputs.ssh_debug_on_failure == true }}
uses: lhotari/action-upterm@v1
with:
## If no one connects shut down ssh server after timeout.
wait-timeout-minutes: 20
wait-timeout-minutes: 20
Loading
Loading