diff --git a/.github/tests/ethlambda-el-all-clients.yaml b/.github/tests/ethlambda-el-all-clients.yaml new file mode 100644 index 000000000..a18da2752 --- /dev/null +++ b/.github/tests/ethlambda-el-all-clients.yaml @@ -0,0 +1,74 @@ +# Cross-client EL × ethlambda experiment: 6 EL clients × 2 each, every +# node paired with ethlambda via Engine API (12 EL+CL pairs total). +# One ethlambda is the aggregator; the remaining 11 are non-aggregators. +# +# Client set: ethrex, nethermind, geth, erigon, nimbus-el (nimbus-eth1), +# besu. ethereumjs and reth omitted from this experiment. +# +# The ethlambda image is built locally from lambdaclass/ethlambda#367 — +# see the PR description for the build command. +# +# Dora intentionally omitted — it requires at least one Eth1 beacon +# endpoint to start, and Lean cl_types don't expose one yet. Re-enable +# once ethlambda ships Beacon API compatibility stubs. +participants: + # First ethrex+ethlambda pair is the network aggregator. + - el_type: ethrex + cl_type: ethlambda + cl_image: ghcr.io/lambdaclass/ethlambda:engine-api-integration + count: 1 + validator_count: 1 + is_aggregator: true + # Second ethrex+ethlambda — non-aggregator peer. + - el_type: ethrex + cl_type: ethlambda + cl_image: ghcr.io/lambdaclass/ethlambda:engine-api-integration + count: 1 + validator_count: 1 + is_aggregator: false + # 2× nethermind + - el_type: nethermind + cl_type: ethlambda + cl_image: ghcr.io/lambdaclass/ethlambda:engine-api-integration + count: 2 + validator_count: 1 + is_aggregator: false + # 2× geth + - el_type: geth + cl_type: ethlambda + cl_image: ghcr.io/lambdaclass/ethlambda:engine-api-integration + count: 2 + validator_count: 1 + is_aggregator: false + # 2× erigon + - el_type: erigon + cl_type: ethlambda + cl_image: ghcr.io/lambdaclass/ethlambda:engine-api-integration + count: 2 + validator_count: 1 + is_aggregator: false + # 2× nimbus-eth1 + - el_type: nimbus + cl_type: ethlambda + cl_image: ghcr.io/lambdaclass/ethlambda:engine-api-integration + count: 2 + validator_count: 1 + is_aggregator: false + # 2× besu + - el_type: besu + cl_type: ethlambda + cl_image: ghcr.io/lambdaclass/ethlambda:engine-api-integration + count: 2 + validator_count: 1 + is_aggregator: false + +lean_network_params: + # 180s gives the 12 EL containers time to warm up before the Lean + # chain starts producing blocks. With the default 60s, several early + # slots missed because proposers couldn't get a payload back from + # their paired EL within the 4s slot window during EL JIT/cache + # warm-up; those early gaps then blocked 3SF-mini's `delta ≤ 5` + # finalization rule. + genesis_delay: 180 + +additional_services: [] diff --git a/.github/tests/ethlambda-el-pair-2node.yaml b/.github/tests/ethlambda-el-pair-2node.yaml new file mode 100644 index 000000000..db2ddad1b --- /dev/null +++ b/.github/tests/ethlambda-el-pair-2node.yaml @@ -0,0 +1,23 @@ +# Two ethrex + ethlambda pairs. One aggregator, one not. +# +# - Two `participants:` entries, each pairs an ethrex EL with an ethlambda CL. +# - Lean genesis pipeline runs once across both participants (single XMSS keyset, +# shared validator-config) — see src/lean/lean_launcher.star. +# - Each ethlambda gets its own EL endpoint, JWT, and EL genesis block hash. +# - Lean libp2p QUIC mesh peers ethlambda_0 <-> ethlambda_1 directly via the +# nodes.yaml the genesis pipeline emits. +participants: + - el_type: ethrex + cl_type: ethlambda + cl_image: ghcr.io/lambdaclass/ethlambda:engine-api-integration + count: 1 + validator_count: 1 + is_aggregator: true + - el_type: ethrex + cl_type: ethlambda + cl_image: ghcr.io/lambdaclass/ethlambda:engine-api-integration + count: 1 + validator_count: 1 + is_aggregator: false + +additional_services: [] diff --git a/.github/tests/ethlambda-el-pair.yaml b/.github/tests/ethlambda-el-pair.yaml new file mode 100644 index 000000000..2ae42c363 --- /dev/null +++ b/.github/tests/ethlambda-el-pair.yaml @@ -0,0 +1,21 @@ +# Single ethrex + ethlambda pair, smoke test for Engine API pairing. +# +# - ethlambda runs as a CL inside `participants:` (cl_type: ethlambda). +# - Its launcher is wired in src/lean/ethlambda/ethlambda_launcher.star; the +# standard CL dispatcher in src/cl/cl_launcher.star skips Lean cl_types, +# then main.star routes them into the Lean pipeline with the paired +# el_context and the network JWT. +# - The `:engine-api-integration` image is built from +# lambdaclass/ethlambda#367 locally (no registry yet). +# +# Other Lean clients live in `participants:` too with `el_type: none` +# (no Engine API yet) — see `lean-devnet4.yaml` and `lean-smoke.yaml`. +participants: + - el_type: ethrex + cl_type: ethlambda + cl_image: ghcr.io/lambdaclass/ethlambda:engine-api-integration + count: 1 + validator_count: 1 + is_aggregator: true + +additional_services: [] diff --git a/.github/tests/lean-devnet4.yaml b/.github/tests/lean-devnet4.yaml new file mode 100644 index 000000000..55a8fedc0 --- /dev/null +++ b/.github/tests/lean-devnet4.yaml @@ -0,0 +1,64 @@ +# 14-node Lean devnet4. Every participant pins the devnet4 image +# explicitly via `cl_image:`; DEFAULT_CL_IMAGES point at `:latest` so +# the defaults don't rot when a new devnet generation ships. +# +# Single aggregator on the first ethlambda. lean_lighthouse is omitted: +# the published `hopinheimer/lighthouse:latest` image is still on the +# single-key (devnet3) GENESIS_VALIDATORS layout and won't reach +# consensus with the devnet4 nodes. +participants: + - el_type: none + cl_type: ethlambda + cl_image: ghcr.io/lambdaclass/ethlambda:devnet4 + count: 1 + validator_count: 1 + is_aggregator: true + - el_type: none + cl_type: ethlambda + cl_image: ghcr.io/lambdaclass/ethlambda:devnet4 + count: 1 + validator_count: 1 + is_aggregator: false + - el_type: none + cl_type: ream + cl_image: ghcr.io/reamlabs/ream:latest-devnet4 + count: 2 + validator_count: 1 + is_aggregator: false + - el_type: none + cl_type: zeam + cl_image: blockblaz/zeam:devnet4 + count: 2 + validator_count: 1 + is_aggregator: false + - el_type: none + cl_type: qlean + cl_image: qdrvm/qlean-mini:devnet-4-amd64 + count: 2 + validator_count: 1 + is_aggregator: false + - el_type: none + cl_type: lantern + cl_image: piertwo/lantern:v0.0.4 + count: 2 + validator_count: 1 + is_aggregator: false + - el_type: none + cl_type: lean_grandine + cl_image: sifrai/lean:devnet-4 + count: 2 + validator_count: 1 + is_aggregator: false + - el_type: none + cl_type: gean + cl_image: ghcr.io/geanlabs/gean:devnet4 + count: 2 + validator_count: 1 + is_aggregator: false + +lean_network_params: + genesis_delay: 180 + active_epoch: 18 + attestation_committee_count: 1 + +additional_services: [] diff --git a/.github/tests/lean-smoke.yaml b/.github/tests/lean-smoke.yaml new file mode 100644 index 000000000..54ff15df3 --- /dev/null +++ b/.github/tests/lean-smoke.yaml @@ -0,0 +1,20 @@ +# Smallest reproducible Lean devnet: one aggregator + one peer, both +# ethlambda, no EL pairing. Comes up in ~3 min including hash-sig +# keygen. Use this as a sanity check before reaching for +# `lean-devnet4.yaml` or `ethlambda-el-pair*.yaml`. +participants: + - el_type: none + cl_type: ethlambda + count: 1 + validator_count: 1 + is_aggregator: true + - el_type: none + cl_type: ethlambda + count: 1 + validator_count: 1 + is_aggregator: false +lean_network_params: + genesis_delay: 60 + active_epoch: 18 + attestation_committee_count: 1 +additional_services: [] diff --git a/docs/architecture.md b/docs/architecture.md index 327a25dd4..30e857578 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -75,6 +75,41 @@ Once CL genesis data and keys have been created, the CL client nodes are started There are only two major difference between CL client and EL client launchers. First, the `cl_client_launcher.launch` method also consumes an `el_context`, because each CL client is connected in a 1:1 relationship with an EL client. Second, because CL clients have keys, the keystore files are passed in to the `launch` function as well. +## Lean Ethereum participants + +Lean Ethereum consensus clients live in the standard `participants:` +block with a Lean `cl_type` — one of `ethlambda`, `ream`, `zeam`, +`qlean`, `lantern`, `gean`, `lean_grandine`, `lean_lighthouse`. The +standard CL dispatcher in [`src/cl/cl_launcher.star`](../src/cl/cl_launcher.star) +skips these; `main.star` then routes them to a parallel pipeline under +[`src/lean/`](../src/lean) that runs its own genesis and starts the +client binary. + +`ethlambda` is the only Lean client that implements Engine API today +([lambdaclass/ethlambda#367](https://github.com/lambdaclass/ethlambda/pull/367)); +it can be paired with any EL the way standard CLs are +(`el_type: ethrex, cl_type: ethlambda` etc.). The other Lean clients +run client-only with `el_type: none` until Engine API ships on their +side too. Lean-only deployments (every participant `el_type: none`) +relax the package's first-participant-must-have-EL guard. + +Genesis uses +[`eth-beacon-genesis leanchain`](https://github.com/ethpandaops/eth-beacon-genesis) +for the chain bundle and +[`blockblaz/hash-sig-cli`](https://hub.docker.com/r/blockblaz/hash-sig-cli) +for XMSS validator keypairs (lives in +[`src/prelaunch_data_generator/lean_genesis/`](../src/prelaunch_data_generator/lean_genesis)). +Per-client launchers translate each Lean record into the client's CLI +(see the contract documented inline in each `_launcher.star`). +Metrics ride a vendored Prometheus + Grafana stack with the upstream +Lean client dashboard pre-loaded. + +Canonical args examples: + +- [`.github/tests/ethlambda-el-pair.yaml`](../.github/tests/ethlambda-el-pair.yaml) — single EL + ethlambda pair +- [`.github/tests/ethlambda-el-pair-2node.yaml`](../.github/tests/ethlambda-el-pair-2node.yaml) — two EL + ethlambda pairs, one aggregator + one not +- [`.github/tests/lean-devnet4.yaml`](../.github/tests/lean-devnet4.yaml) — devnet4 multi-client (all `el_type: none`) + ## Auxiliary Services After the Ethereum network is up and running, this package starts several auxiliary containers to make it easier to work with the Ethereum network. At time of writing, these are: diff --git a/main.star b/main.star index 610d0fd4a..2d3dcdb2e 100644 --- a/main.star +++ b/main.star @@ -1,6 +1,7 @@ input_parser = import_module("./src/package_io/input_parser.star") constants = import_module("./src/package_io/constants.star") participant_network = import_module("./src/participant_network.star") +lean_launcher = import_module("./src/lean/lean_launcher.star") shared_utils = import_module("./src/shared_utils/shared_utils.star") static_files = import_module("./src/static_files/static_files.star") genesis_constants = import_module( @@ -329,6 +330,64 @@ def run(plan, args={}): ) all_xatu_sentry_contexts.append(participant.xatu_sentry_context) + # Launch Lean Ethereum consensus participants. Lean clients are + # `participants:` entries whose `cl_type` is in + # constants.LEAN_CL_TYPES. The cl_launcher dispatcher already + # skipped them (None cl_context); here we build equivalent Lean + # records from each participant and hand them — together with the + # paired el_context (None when el_type is `none`) and the network + # JWT — to src/lean/lean_launcher.launch. ethlambda is the only + # Lean client that wires Engine API today (lambdaclass/ethlambda#367); + # the others run with `el_type: none`. + lean_records = [] + for index, participant in enumerate(args_with_right_defaults.participants): + if participant.cl_type not in constants.LEAN_CL_TYPES: + continue + paired_el_context = ( + all_el_contexts[index] if index < len(all_el_contexts) else None + ) + lean_records.append( + { + "lean_type": participant.cl_type, + "lean_image": participant.cl_image, + # The standard input parser already expanded `count: N` into + # N separate `participants:` entries (input_parser.star:1260), + # so we synthesize one Lean record per expanded participant + # and hardcode count=1 here — otherwise the Lean launcher's + # own count expansion would multiply N×N. + "count": 1, + "validator_count": participant.validator_count + or args_with_right_defaults.lean_network_params[ + "num_validator_keys_per_node" + ], + "is_aggregator": participant.is_aggregator, + "lean_extra_params": participant.cl_extra_params, + "lean_extra_env_vars": participant.cl_extra_env_vars, + "lean_extra_labels": participant.cl_extra_labels, + "lean_log_level": participant.cl_log_level, + "lean_min_cpu": participant.cl_min_cpu, + "lean_max_cpu": participant.cl_max_cpu, + "lean_min_mem": participant.cl_min_mem, + "lean_max_mem": participant.cl_max_mem, + "node_selectors": participant.node_selectors, + "tolerations": participant.tolerations, + "prometheus_config": { + "scrape_interval": participant.prometheus_config.scrape_interval, + "labels": participant.prometheus_config.labels or {}, + }, + # Underscore-prefixed fields are internal hand-offs to + # the Lean launcher. + "_el_context": paired_el_context, + } + ) + + all_lean_contexts = lean_launcher.launch( + plan, + lean_records, + args_with_right_defaults.lean_network_params, + jwt_file=jwt_file, + ) + # Generate validator ranges validator_ranges_config_template = read_file( static_files.VALIDATOR_RANGES_CONFIG_TEMPLATE_FILEPATH @@ -340,10 +399,17 @@ def run(plan, args={}): args_with_right_defaults.participants, ) - fuzz_target = "http://{0}:{1}".format( - all_el_contexts[0].ip_addr, - all_el_contexts[0].rpc_port_num, - ) + # `fuzz_target` is only consumed by additional services that talk to an + # Eth1 EL (tx-fuzz, rakoon, broadcaster, custom_flood). For all-Lean + # deployments (every participant `el_type: none`) there is no EL to + # point at; leave it empty and let the additional-service handlers + # guard their own use. + fuzz_target = "" + if len(all_el_contexts) > 0 and all_el_contexts[0] != None: + fuzz_target = "http://{0}:{1}".format( + all_el_contexts[0].ip_addr, + all_el_contexts[0].rpc_port_num, + ) # Broadcaster forwards requests, sent to it, to all nodes in parallel if "broadcaster" in args_with_right_defaults.additional_services: diff --git a/network_params.yaml b/network_params.yaml index 7756bc257..0c1911446 100644 --- a/network_params.yaml +++ b/network_params.yaml @@ -291,3 +291,33 @@ port_publisher: # #!/bin/bash # echo "Hello" extra_files: {} + +# Lean Ethereum knobs (post-quantum signatures, XMSS validator keys, +# leanchain genesis). Lean clients themselves live in the standard +# `participants:` block above with a Lean cl_type (ethlambda, ream, zeam, +# qlean, lantern, gean, lean_grandine, lean_lighthouse). ethlambda is the +# only Lean client that wires Engine API today +# (lambdaclass/ethlambda#367); the rest run with `el_type: none`. +# +# Example - 4 ethlambda nodes paired with ethrex: +# participants: +# - el_type: ethrex +# cl_type: ethlambda +# cl_image: ghcr.io/lambdaclass/ethlambda:engine-api-integration +# count: 4 +# validator_count: 1 +# is_aggregator: true +lean_network_params: + # Seconds added to "now" to compute GENESIS_TIME when genesis_time is 0. + genesis_delay: 60 + # Explicit Unix timestamp; 0 = derive from genesis_delay. + genesis_time: 0 + # leanSpec ATTESTATION_COMMITTEE_COUNT (1 = single committee per slot). + attestation_committee_count: 1 + # log_2(active epochs) for the XMSS hash-sig scheme. + active_epoch: 18 + # Default validator keys per node (overridable per participant). + num_validator_keys_per_node: 1 + # Override the Lean genesis tooling images. Empty = pinned default. + genesis_generator_image: "" + hash_sig_cli_image: "" diff --git a/src/cl/cl_launcher.star b/src/cl/cl_launcher.star index 4011ea22c..849141fc8 100644 --- a/src/cl/cl_launcher.star +++ b/src/cl/cl_launcher.star @@ -147,6 +147,19 @@ def launch( for index, participant in enumerate(args_with_right_defaults.participants): cl_type = participant.cl_type el_type = participant.el_type + + # Lean cl_types are launched by the Lean pipeline (src/lean/), + # not by the standard CL dispatcher. We append `None` to the + # per-index context lists so downstream `all_cl_contexts[index]` + # lookups stay aligned with `args_with_right_defaults.participants`; + # main.star then runs the Lean launcher with the right el_contexts + # and jwt_file. Mirrors the EL_TYPE.none / cl-only path the package + # already supports for `consensoor` etc. + if cl_type in constants.LEAN_CL_TYPES: + all_cl_contexts.append(None) + all_snooper_el_engine_contexts.append(None) + continue + node_selectors = input_parser.get_client_node_selectors( participant.node_selectors, global_node_selectors, diff --git a/src/dora/dora_launcher.star b/src/dora/dora_launcher.star index 223f9d07f..3f251e186 100644 --- a/src/dora/dora_launcher.star +++ b/src/dora/dora_launcher.star @@ -48,6 +48,11 @@ def launch_dora( all_cl_client_info = [] all_el_client_info = [] for index, participant in enumerate(participant_contexts): + # Skip Lean cl_types entirely — they don't expose an Eth1 beacon + # API, so they can't be added to either the CL or EL lists that + # dora's config template renders against. + if participant_configs[index].cl_type in constants.LEAN_CL_TYPES: + continue full_name, cl_client, el_client, _ = shared_utils.get_client_names( participant, index, participant_contexts, participant_configs ) diff --git a/src/el/ethrex/ethrex_launcher.star b/src/el/ethrex/ethrex_launcher.star index 1d49d8657..3af73f296 100644 --- a/src/el/ethrex/ethrex_launcher.star +++ b/src/el/ethrex/ethrex_launcher.star @@ -162,6 +162,10 @@ def get_config( "--log.level={0}".format(VERBOSITY_LEVELS[global_log_level]), "--http.port={0}".format(RPC_PORT_NUM), "--http.addr=0.0.0.0", + # Enable the `admin` namespace so el_admin_node_info.get_enode_enr_for_node() + # can read the ENR/enode via `admin_nodeInfo`. ethrex defaults to + # `eth,net,web3` only; without `admin` the poll hangs forever. + "--http.api=admin,eth,net,web3", "--authrpc.port={0}".format(ENGINE_RPC_PORT_NUM), "--authrpc.jwtsecret=" + constants.JWT_MOUNT_PATH_ON_CONTAINER, "--authrpc.addr=0.0.0.0", diff --git a/src/lean/ethlambda/ethlambda_launcher.star b/src/lean/ethlambda/ethlambda_launcher.star new file mode 100644 index 000000000..e8861a739 --- /dev/null +++ b/src/lean/ethlambda/ethlambda_launcher.star @@ -0,0 +1,252 @@ +""" +ethlambda launcher. + +Translates the Lean pipeline's per-node record into the ethlambda CLI surface +(see lambdaclass/ethlambda's `bin/ethlambda/src/main.rs` and the matching +lean-quickstart `client-cmds/ethlambda-cmd.sh`). + +Lifecycle (called by ../lean_launcher.star): + 1. `initialize()` - add a Kurtosis service mounting both the P2P keys + artifact and the hash-sig (XMSS) keys artifact (neither depends on the + node IP). The container runs `tail -f` on a log file so Kurtosis keeps + the service alive while the genesis pipeline computes things downstream + that DO need the IP. + 2. `start()` - after the genesis tool has produced the text artifacts + (config.yaml, annotated_validators.yaml, nodes.yaml, validator-config.yaml), + stage each text file inside the running container via `plan.exec` + (Kurtosis doesn't let us mount new file artifacts onto a running + service), then launch the ethlambda binary as a backgrounded `nohup` + process. We use `plan.exec` for both steps rather than a second + `add_service`, because the Starlark validator rejects two + `add_service` calls with the same name even when `force_update=True`. + +The hash-sig keys, P2P keys, and a stable directory layout under +`/network-configs/` are established by `initialize`; `start` only adds the +files whose contents depend on the live IP allocation. +""" + +constants = import_module("../../package_io/constants.star") +lean_shared = import_module("../lean_shared.star") +lean_context = import_module("../lean_context.star") + +ENTRYPOINT = "/usr/local/bin/ethlambda" +GENESIS_MOUNT = constants.LEAN_GENESIS_MOUNTPOINT_ON_CLIENTS +HASH_SIG_MOUNT = GENESIS_MOUNT + "/hash-sig-keys" +DATA_DIR = "/data" +NODE_KEY_MOUNT = constants.LEAN_NODE_KEY_MOUNTPOINT_ON_CLIENTS + +GENESIS_TEXT_FILES = [ + "config.yaml", + "annotated_validators.yaml", + "nodes.yaml", + "validator-config.yaml", +] + + +def initialize(plan, node, p2p_keys_artifact, hash_sig_artifact): + """Phase 1: stand the placeholder service up so Kurtosis assigns an IP. + + Both the P2P keys and the XMSS keys are mounted here because neither + depends on the IP allocation. Genesis text files (config.yaml, + nodes.yaml, etc.) are staged later via `plan.exec` once the genesis + tool has computed them against the live IPs. + """ + cfg_kwargs = lean_shared.common_cfg_kwargs(node) + cfg_kwargs.update( + { + "image": node["image"], + # Override the image ENTRYPOINT so the container doesn't try to + # start ethlambda with no flags before we've written the genesis + # files in. + "entrypoint": ["/bin/sh", "-c"], + "cmd": lean_shared.lean_tail_logs_cmd(node["service_name"])[2:], + "files": { + NODE_KEY_MOUNT: p2p_keys_artifact, + HASH_SIG_MOUNT: hash_sig_artifact, + }, + } + ) + return plan.add_service(node["service_name"], ServiceConfig(**cfg_kwargs)) + + +def start( + plan, + node, + service, + genesis_artifact, + hash_sig_artifact, + el_context=None, + jwt_file=None, + el_genesis_block_hash=None, +): + """Phase 3: stage genesis text files into the running container and + launch the ethlambda binary as a backgrounded process. + + `hash_sig_artifact` is unused here because it was already mounted in + `initialize()` — we keep the parameter for shape parity with the other + Lean clients. + + When `el_context`, `jwt_file`, and `el_genesis_block_hash` are all set, + ethlambda is launched with Engine API pairing + (`--execution-endpoint`, `--execution-jwt-secret`, + `--execution-genesis-block-hash`). Otherwise it boots Lean-only. + See lambdaclass/ethlambda#367 for the Engine API plumbing. + """ + service_name = service.name + log_file = lean_shared.lean_log_file_path(service_name) + + # Stage each text genesis file inside the running container. We read + # the artifact's contents via plan.run_sh (the output is a Kurtosis + # runtime future) and then plan.exec a `cat > path` heredoc that + # carries the future as a string argument — Kurtosis resolves the + # future at apply time so the literal file contents land in the + # target. mkdir is idempotent. + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=["/bin/sh", "-c", "mkdir -p {0}".format(GENESIS_MOUNT)], + ), + description="Preparing genesis mount on {0}".format(service_name), + ) + for filename in GENESIS_TEXT_FILES: + read = plan.run_sh( + run="cat /src/{0}".format(filename), + files={"/src": genesis_artifact}, + description="Reading {0} for {1}".format(filename, service_name), + ) + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=[ + "/bin/sh", + "-c", + "cat > {0}/{1} <<'ETHLAMBDA_EOF'\n{2}\nETHLAMBDA_EOF".format( + GENESIS_MOUNT, + filename, + read.output, + ), + ], + ), + description="Staging {0} into {1}".format(filename, service_name), + ) + + # Build and start the ethlambda command. The placeholder `tail -f` + # keeps the container alive, so we run ethlambda as a nohup + # background process and let its stdout/stderr flow into the same + # log file the tail is already watching. + cmd_parts = [ + ENTRYPOINT, + "--genesis", + "{0}/config.yaml".format(GENESIS_MOUNT), + "--validators", + "{0}/annotated_validators.yaml".format(GENESIS_MOUNT), + "--bootnodes", + "{0}/nodes.yaml".format(GENESIS_MOUNT), + "--validator-config", + "{0}/validator-config.yaml".format(GENESIS_MOUNT), + "--hash-sig-keys-dir", + HASH_SIG_MOUNT, + "--data-dir", + DATA_DIR, + "--gossipsub-port", + str(constants.LEAN_QUIC_PORT_NUM), + "--node-id", + node["node_name"], + "--node-key", + "{0}/{1}.key".format(NODE_KEY_MOUNT, node["node_name"]), + "--http-address", + "0.0.0.0", + "--api-port", + str(constants.LEAN_API_PORT_NUM), + "--metrics-port", + str(constants.LEAN_METRICS_PORT_NUM), + ] + if node["is_aggregator"]: + cmd_parts.append("--is-aggregator") + + # Engine API pairing: present only when this node was synthesized from + # a `participants:` entry with a paired EL (lean_launcher fills the + # three values in tandem). We stage the JWT secret into the running + # container via plan.exec (same trick as the genesis files) since + # Kurtosis doesn't let us add a new files mount to an already-running + # service. + if el_context != None and jwt_file != None and el_genesis_block_hash != None: + jwt_path = "{0}/jwtsecret".format(GENESIS_MOUNT) + jwt_read = plan.run_sh( + run="cat /src/jwtsecret", + files={"/src": jwt_file}, + description="Reading JWT secret for {0}".format(node["service_name"]), + ) + plan.exec( + service_name=node["service_name"], + recipe=ExecRecipe( + command=[ + "/bin/sh", + "-c", + "cat > {0} <<'ETHLAMBDA_JWT_EOF'\n{1}\nETHLAMBDA_JWT_EOF".format( + jwt_path, + jwt_read.output, + ), + ], + ), + description="Staging JWT into {0}".format(node["service_name"]), + ) + engine_endpoint = "http://{0}:{1}".format( + el_context.ip_addr, el_context.engine_rpc_port_num + ) + cmd_parts.extend( + [ + "--execution-endpoint", + engine_endpoint, + "--execution-jwt-secret", + jwt_path, + "--execution-genesis-block-hash", + el_genesis_block_hash, + ] + ) + + for extra in node["extra_params"]: + cmd_parts.append(extra) + + rust_log = "" + if node["log_level"] != "": + rust_log = "RUST_LOG='{0}' ".format(node["log_level"]) + + nohup_cmd = "setsid -f sh -c \"exec {0}{1}\" < /dev/null >> {2} 2>&1".format( + rust_log, + " ".join(cmd_parts), + log_file, + ) + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=["/bin/sh", "-c", nohup_cmd], + ), + description="Starting ethlambda binary on {0}".format(service_name), + ) + + api_url = "http://{0}:{1}".format( + service.ip_address, + constants.LEAN_API_PORT_NUM, + ) + metrics_url = "http://{0}:{1}/metrics".format( + service.ip_address, + constants.LEAN_METRICS_PORT_NUM, + ) + + return lean_context.new_lean_context( + client_name=constants.LEAN_TYPE.ethlambda, + service_name=service_name, + ip_address=service.ip_address, + quic_port=constants.LEAN_QUIC_PORT_NUM, + api_port=constants.LEAN_API_PORT_NUM, + metrics_port=constants.LEAN_METRICS_PORT_NUM, + api_url=api_url, + metrics_url=metrics_url, + metrics_info={ + "name": service_name, + "url": metrics_url, + "path": "/metrics", + "config": node["prometheus_config"], + }, + ) diff --git a/src/lean/gean/gean_launcher.star b/src/lean/gean/gean_launcher.star new file mode 100644 index 000000000..a8ffe2653 --- /dev/null +++ b/src/lean/gean/gean_launcher.star @@ -0,0 +1,156 @@ +""" +gean launcher. + +Translates the Lean pipeline's per-node record into gean's CLI surface +from `client-cmds/gean-cmd.sh` in blockblaz/lean-quickstart: + + gean \ + --custom-network-config-dir \ + --gossipsub-port \ + --node-id --node-key \ + --http-address 0.0.0.0 --api-port \ + --metrics-port +""" + +constants = import_module("../../package_io/constants.star") +lean_shared = import_module("../lean_shared.star") +lean_context = import_module("../lean_context.star") + +ENTRYPOINT = "/usr/local/bin/gean" +GENESIS_MOUNT = constants.LEAN_GENESIS_MOUNTPOINT_ON_CLIENTS +HASH_SIG_MOUNT = GENESIS_MOUNT + "/hash-sig-keys" +DATA_DIR = "/data" +NODE_KEY_MOUNT = constants.LEAN_NODE_KEY_MOUNTPOINT_ON_CLIENTS + +GENESIS_TEXT_FILES = [ + "config.yaml", + "annotated_validators.yaml", + "nodes.yaml", + "validator-config.yaml", + "validators.yaml", +] + + +def initialize(plan, node, p2p_keys_artifact, hash_sig_artifact): + cfg_kwargs = lean_shared.common_cfg_kwargs(node) + cfg_kwargs.update( + { + "image": node["image"], + "entrypoint": ["/bin/sh", "-c"], + "cmd": lean_shared.lean_tail_logs_cmd(node["service_name"])[2:], + "files": { + NODE_KEY_MOUNT: p2p_keys_artifact, + HASH_SIG_MOUNT: hash_sig_artifact, + }, + } + ) + return plan.add_service(node["service_name"], ServiceConfig(**cfg_kwargs)) + + +def start(plan, node, service, genesis_artifact, hash_sig_artifact): + service_name = service.name + log_file = lean_shared.lean_log_file_path(service_name) + + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=["/bin/sh", "-c", "mkdir -p {0}".format(GENESIS_MOUNT)], + ), + description="Preparing genesis mount on {0}".format(service_name), + ) + for filename in GENESIS_TEXT_FILES: + read = plan.run_sh( + run="cat /src/{0}".format(filename), + files={"/src": genesis_artifact}, + description="Reading {0} for {1}".format(filename, service_name), + ) + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=[ + "/bin/sh", + "-c", + "cat > {0}/{1} <<'GEAN_EOF'\n{2}\nGEAN_EOF".format( + GENESIS_MOUNT, + filename, + read.output, + ), + ], + ), + description="Staging {0} into {1}".format(filename, service_name), + ) + + # gean's contract: --custom-network-config-dir points at a dir, and + # --node-key is resolved relative to it as `.key`. Stage the + # P2P key into the genesis dir so the conventional layout works. + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=[ + "/bin/sh", + "-c", + "cp {0}/{1}.key {2}/{1}.key".format( + NODE_KEY_MOUNT, + node["node_name"], + GENESIS_MOUNT, + ), + ], + ), + description="Staging node key into {0}".format(service_name), + ) + + cmd_parts = [ + ENTRYPOINT, + "--custom-network-config-dir", + GENESIS_MOUNT, + "--gossipsub-port", + str(constants.LEAN_QUIC_PORT_NUM), + "--node-id", + node["node_name"], + "--node-key", + "{0}/{1}.key".format(GENESIS_MOUNT, node["node_name"]), + "--http-address", + "0.0.0.0", + "--api-port", + str(constants.LEAN_API_PORT_NUM), + "--metrics-port", + str(constants.LEAN_METRICS_PORT_NUM), + ] + if node["is_aggregator"]: + cmd_parts.append("--is-aggregator") + for extra in node["extra_params"]: + cmd_parts.append(extra) + + nohup_cmd = "setsid -f sh -c \"exec {0}\" < /dev/null >> {1} 2>&1".format( + " ".join(cmd_parts), + log_file, + ) + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=["/bin/sh", "-c", nohup_cmd], + ), + description="Starting gean binary on {0}".format(service_name), + ) + + api_url = "http://{0}:{1}".format(service.ip_address, constants.LEAN_API_PORT_NUM) + metrics_url = "http://{0}:{1}/metrics".format( + service.ip_address, constants.LEAN_METRICS_PORT_NUM + ) + + return lean_context.new_lean_context( + client_name=constants.LEAN_TYPE.gean, + service_name=service_name, + ip_address=service.ip_address, + quic_port=constants.LEAN_QUIC_PORT_NUM, + api_port=constants.LEAN_API_PORT_NUM, + metrics_port=constants.LEAN_METRICS_PORT_NUM, + api_url=api_url, + metrics_url=metrics_url, + metrics_info={ + "name": service_name, + "url": metrics_url, + "path": "/metrics", + "config": node["prometheus_config"], + }, + ) diff --git a/src/lean/grandine/grandine_launcher.star b/src/lean/grandine/grandine_launcher.star new file mode 100644 index 000000000..1834c6bbb --- /dev/null +++ b/src/lean/grandine/grandine_launcher.star @@ -0,0 +1,142 @@ +""" +grandine (lean) launcher. + +Translates the Lean pipeline's per-node record into grandine's Lean +client CLI from `client-cmds/grandine-cmd.sh` in blockblaz/lean-quickstart. +""" + +constants = import_module("../../package_io/constants.star") +lean_shared = import_module("../lean_shared.star") +lean_context = import_module("../lean_context.star") + +# Image ENTRYPOINT is `lean_client`. We bypass it via `/bin/sh -c` and +# call the resolved binary path directly. +ENTRYPOINT = "/usr/local/bin/lean_client" +GENESIS_MOUNT = constants.LEAN_GENESIS_MOUNTPOINT_ON_CLIENTS +HASH_SIG_MOUNT = GENESIS_MOUNT + "/hash-sig-keys" +DATA_DIR = "/data" +NODE_KEY_MOUNT = constants.LEAN_NODE_KEY_MOUNTPOINT_ON_CLIENTS + +GENESIS_TEXT_FILES = [ + "config.yaml", + "annotated_validators.yaml", + "nodes.yaml", + "validator-config.yaml", +] + + +def initialize(plan, node, p2p_keys_artifact, hash_sig_artifact): + cfg_kwargs = lean_shared.common_cfg_kwargs(node) + cfg_kwargs.update( + { + "image": node["image"], + "entrypoint": ["/bin/sh", "-c"], + "cmd": lean_shared.lean_tail_logs_cmd(node["service_name"])[2:], + "files": { + NODE_KEY_MOUNT: p2p_keys_artifact, + HASH_SIG_MOUNT: hash_sig_artifact, + }, + } + ) + return plan.add_service(node["service_name"], ServiceConfig(**cfg_kwargs)) + + +def start(plan, node, service, genesis_artifact, hash_sig_artifact): + service_name = service.name + log_file = lean_shared.lean_log_file_path(service_name) + + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=["/bin/sh", "-c", "mkdir -p {0}".format(GENESIS_MOUNT)], + ), + description="Preparing genesis mount on {0}".format(service_name), + ) + for filename in GENESIS_TEXT_FILES: + read = plan.run_sh( + run="cat /src/{0}".format(filename), + files={"/src": genesis_artifact}, + description="Reading {0} for {1}".format(filename, service_name), + ) + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=[ + "/bin/sh", + "-c", + "cat > {0}/{1} <<'GRANDINE_EOF'\n{2}\nGRANDINE_EOF".format( + GENESIS_MOUNT, + filename, + read.output, + ), + ], + ), + description="Staging {0} into {1}".format(filename, service_name), + ) + + cmd_parts = [ + ENTRYPOINT, + "--genesis", + "{0}/config.yaml".format(GENESIS_MOUNT), + "--validator-registry-path", + "{0}/annotated_validators.yaml".format(GENESIS_MOUNT), + "--bootnodes", + "{0}/nodes.yaml".format(GENESIS_MOUNT), + "--node-id", + node["node_name"], + "--node-key", + "{0}/{1}.key".format(NODE_KEY_MOUNT, node["node_name"]), + "--port", + str(constants.LEAN_QUIC_PORT_NUM), + "--address", + "0.0.0.0", + "--http-address", + "0.0.0.0", + "--http-port", + str(constants.LEAN_API_PORT_NUM), + "--metrics", + "--metrics-address", + "0.0.0.0", + "--metrics-port", + str(constants.LEAN_METRICS_PORT_NUM), + "--hash-sig-key-dir", + HASH_SIG_MOUNT, + ] + if node["is_aggregator"]: + cmd_parts.append("--is-aggregator") + for extra in node["extra_params"]: + cmd_parts.append(extra) + + nohup_cmd = "setsid -f sh -c \"exec {0}\" < /dev/null >> {1} 2>&1".format( + " ".join(cmd_parts), + log_file, + ) + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=["/bin/sh", "-c", nohup_cmd], + ), + description="Starting grandine binary on {0}".format(service_name), + ) + + api_url = "http://{0}:{1}".format(service.ip_address, constants.LEAN_API_PORT_NUM) + metrics_url = "http://{0}:{1}/metrics".format( + service.ip_address, constants.LEAN_METRICS_PORT_NUM + ) + + return lean_context.new_lean_context( + client_name=constants.LEAN_TYPE.grandine, + service_name=service_name, + ip_address=service.ip_address, + quic_port=constants.LEAN_QUIC_PORT_NUM, + api_port=constants.LEAN_API_PORT_NUM, + metrics_port=constants.LEAN_METRICS_PORT_NUM, + api_url=api_url, + metrics_url=metrics_url, + metrics_info={ + "name": service_name, + "url": metrics_url, + "path": "/metrics", + "config": node["prometheus_config"], + }, + ) diff --git a/src/lean/lantern/lantern_launcher.star b/src/lean/lantern/lantern_launcher.star new file mode 100644 index 000000000..34f40c811 --- /dev/null +++ b/src/lean/lantern/lantern_launcher.star @@ -0,0 +1,147 @@ +""" +lantern launcher. + +Translates the Lean pipeline's per-node record into lantern's CLI surface +from `client-cmds/lantern-cmd.sh` in blockblaz/lean-quickstart. +""" + +constants = import_module("../../package_io/constants.star") +lean_shared = import_module("../lean_shared.star") +lean_context = import_module("../lean_context.star") + +# The image's ENTRYPOINT is /usr/local/bin/lantern-entrypoint.sh; we +# bypass it via `entrypoint: ["/bin/sh", "-c"]` and call the real binary +# directly. lean-quickstart calls it as `lantern_cli` for the binary path +# but the entrypoint script forwards to the same underlying binary; the +# in-container path is /usr/local/bin/lantern_cli. +ENTRYPOINT = "/opt/lantern/bin/lantern" +GENESIS_MOUNT = constants.LEAN_GENESIS_MOUNTPOINT_ON_CLIENTS +HASH_SIG_MOUNT = GENESIS_MOUNT + "/hash-sig-keys" +DATA_DIR = "/data" +NODE_KEY_MOUNT = constants.LEAN_NODE_KEY_MOUNTPOINT_ON_CLIENTS + +GENESIS_TEXT_FILES = [ + "config.yaml", + "annotated_validators.yaml", + "nodes.yaml", + "validator-config.yaml", + "validators.yaml", +] + + +def initialize(plan, node, p2p_keys_artifact, hash_sig_artifact): + cfg_kwargs = lean_shared.common_cfg_kwargs(node) + cfg_kwargs.update( + { + "image": node["image"], + "entrypoint": ["/bin/sh", "-c"], + "cmd": lean_shared.lean_tail_logs_cmd(node["service_name"])[2:], + "files": { + NODE_KEY_MOUNT: p2p_keys_artifact, + HASH_SIG_MOUNT: hash_sig_artifact, + }, + } + ) + return plan.add_service(node["service_name"], ServiceConfig(**cfg_kwargs)) + + +def start(plan, node, service, genesis_artifact, hash_sig_artifact): + service_name = service.name + log_file = lean_shared.lean_log_file_path(service_name) + + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=["/bin/sh", "-c", "mkdir -p {0}".format(GENESIS_MOUNT)], + ), + description="Preparing genesis mount on {0}".format(service_name), + ) + for filename in GENESIS_TEXT_FILES: + read = plan.run_sh( + run="cat /src/{0}".format(filename), + files={"/src": genesis_artifact}, + description="Reading {0} for {1}".format(filename, service_name), + ) + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=[ + "/bin/sh", + "-c", + "cat > {0}/{1} <<'LANTERN_EOF'\n{2}\nLANTERN_EOF".format( + GENESIS_MOUNT, + filename, + read.output, + ), + ], + ), + description="Staging {0} into {1}".format(filename, service_name), + ) + + cmd_parts = [ + ENTRYPOINT, + "--data-dir", + DATA_DIR, + "--genesis-config", + "{0}/config.yaml".format(GENESIS_MOUNT), + "--validator-registry-path", + "{0}/validators.yaml".format(GENESIS_MOUNT), + "--validator-keys-path", + "{0}/annotated_validators.yaml".format(GENESIS_MOUNT), + "--validator-config", + "{0}/validator-config.yaml".format(GENESIS_MOUNT), + "--nodes-path", + "{0}/nodes.yaml".format(GENESIS_MOUNT), + "--node-id", + node["node_name"], + "--node-key-path", + "{0}/{1}.key".format(NODE_KEY_MOUNT, node["node_name"]), + "--listen-address", + "/ip4/0.0.0.0/udp/{0}/quic-v1".format(constants.LEAN_QUIC_PORT_NUM), + "--metrics-port", + str(constants.LEAN_METRICS_PORT_NUM), + "--http-port", + str(constants.LEAN_API_PORT_NUM), + "--log-level", + "info", + "--hash-sig-key-dir", + HASH_SIG_MOUNT, + ] + if node["is_aggregator"]: + cmd_parts.append("--is-aggregator") + for extra in node["extra_params"]: + cmd_parts.append(extra) + + nohup_cmd = "setsid -f sh -c \"exec {0}\" < /dev/null >> {1} 2>&1".format( + " ".join(cmd_parts), + log_file, + ) + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=["/bin/sh", "-c", nohup_cmd], + ), + description="Starting lantern binary on {0}".format(service_name), + ) + + api_url = "http://{0}:{1}".format(service.ip_address, constants.LEAN_API_PORT_NUM) + metrics_url = "http://{0}:{1}/metrics".format( + service.ip_address, constants.LEAN_METRICS_PORT_NUM + ) + + return lean_context.new_lean_context( + client_name=constants.LEAN_TYPE.lantern, + service_name=service_name, + ip_address=service.ip_address, + quic_port=constants.LEAN_QUIC_PORT_NUM, + api_port=constants.LEAN_API_PORT_NUM, + metrics_port=constants.LEAN_METRICS_PORT_NUM, + api_url=api_url, + metrics_url=metrics_url, + metrics_info={ + "name": service_name, + "url": metrics_url, + "path": "/metrics", + "config": node["prometheus_config"], + }, + ) diff --git a/src/lean/lean_context.star b/src/lean/lean_context.star new file mode 100644 index 000000000..bfeb8f4e6 --- /dev/null +++ b/src/lean/lean_context.star @@ -0,0 +1,30 @@ +""" +Per-node Lean context. + +Returned by every `lean//_launcher.start_*` and consumed by +Prometheus / Grafana / dora / etc. to discover the running Lean nodes. +""" + + +def new_lean_context( + client_name, + service_name, + ip_address, + quic_port, + api_port, + metrics_port, + api_url, + metrics_url, + metrics_info=None, +): + return struct( + client_name=client_name, + service_name=service_name, + ip_address=ip_address, + quic_port=quic_port, + api_port=api_port, + metrics_port=metrics_port, + api_url=api_url, + metrics_url=metrics_url, + metrics_info=metrics_info, + ) diff --git a/src/lean/lean_launcher.star b/src/lean/lean_launcher.star new file mode 100644 index 000000000..0a0965aba --- /dev/null +++ b/src/lean/lean_launcher.star @@ -0,0 +1,282 @@ +""" +Lean Ethereum participant launcher. + +Orchestrates the entire Lean pipeline: + + 1. Generate per-node libp2p P2P keys (so we can render ENRs deterministically). + 2. Initialise placeholder Kurtosis services so we get assigned IPs. + 3. Run the Lean genesis pipeline (eth-beacon-genesis leanchain + hash-sig-cli) + against the live IPs. + 4. Mount the genesis bundle into each placeholder and start the real client + binary via `plan.exec`. + +Invoked from main.star with a list of Lean records derived from +`participants:` entries whose `cl_type` is in constants.LEAN_CL_TYPES. +Each record carries the paired `el_context` (None when `el_type: none`) +so the launcher can wire Engine API for clients that implement it (today +just ethlambda — lambdaclass/ethlambda#367). +""" + +constants = import_module("../package_io/constants.star") +lean_shared = import_module("./lean_shared.star") +lean_genesis = import_module( + "../prelaunch_data_generator/lean_genesis/lean_genesis_generator.star" +) +p2p_keys = import_module( + "../prelaunch_data_generator/lean_genesis/p2p_keys_generator.star" +) + +ethlambda_launcher = import_module("./ethlambda/ethlambda_launcher.star") +ream_launcher = import_module("./ream/ream_launcher.star") +zeam_launcher = import_module("./zeam/zeam_launcher.star") +qlean_launcher = import_module("./qlean/qlean_launcher.star") +lantern_launcher = import_module("./lantern/lantern_launcher.star") +grandine_launcher = import_module("./grandine/grandine_launcher.star") +lighthouse_launcher = import_module("./lighthouse/lighthouse_launcher.star") +gean_launcher = import_module("./gean/gean_launcher.star") +metrics_launcher = import_module("./metrics/metrics_launcher.star") + + +def _launcher_for(lean_type): + if lean_type == constants.LEAN_TYPE.ethlambda: + return ethlambda_launcher + elif lean_type == constants.LEAN_TYPE.ream: + return ream_launcher + elif lean_type == constants.LEAN_TYPE.zeam: + return zeam_launcher + elif lean_type == constants.LEAN_TYPE.qlean: + return qlean_launcher + elif lean_type == constants.LEAN_TYPE.lantern: + return lantern_launcher + elif lean_type == constants.LEAN_TYPE.grandine: + return grandine_launcher + elif lean_type == constants.LEAN_TYPE.lighthouse: + return lighthouse_launcher + elif lean_type == constants.LEAN_TYPE.gean: + return gean_launcher + fail( + "Unsupported lean_type '{0}'. Supported: {1}. See ".format( + lean_type, + ", ".join( + [ + constants.LEAN_TYPE.ethlambda, + constants.LEAN_TYPE.ream, + constants.LEAN_TYPE.zeam, + constants.LEAN_TYPE.qlean, + constants.LEAN_TYPE.lantern, + constants.LEAN_TYPE.grandine, + constants.LEAN_TYPE.lighthouse, + constants.LEAN_TYPE.gean, + ] + ), + ) + + "docs/lean-adding-a-new-client.md to add a new client." + ) + + +def launch(plan, lean_participants, lean_network_params, jwt_file=None): + """Top-level entrypoint for the Lean pipeline. + + Returns the list of `lean_context` structs (one per running node), + suitable for handing to Prometheus / Grafana / dora. + + `jwt_file` is the JWT artifact shared with paired ELs. It's only + consulted for nodes that carry an `_el_context` (synthesized by + main.star from `participants:` entries with a Lean cl_type). + """ + if not lean_participants: + return [] + + # Expand per-participant `count` to a flat list of (type, image, ...) records. + # Naming follows lean-quickstart's `_` convention so a + # Lean client's existing log parsers and dashboards work unchanged. + expanded = [] + type_counters = {} + for participant in lean_participants: + lean_type = participant["lean_type"] + for _ in range(participant["count"]): + idx = type_counters.get(lean_type, 0) + type_counters[lean_type] = idx + 1 + # `node_name` follows lean-quickstart's `_` + # convention (passed as --node-id and used in validator-config.yaml). + # Kurtosis service names, however, must match RFC 1035 — lowercase + # letters/digits/hyphens only — so we translate any underscore in + # the lean_type (e.g. `lean_grandine`) to a hyphen for the + # Kurtosis-facing service name. + node_name = "{0}_{1}".format(lean_type, idx) + service_name = "lean-{0}-{1}".format( + lean_type.replace("_", "-"), idx + ) + expanded.append( + { + "node_name": node_name, + "service_name": service_name, + "lean_type": lean_type, + "image": participant["lean_image"], + "validator_count": participant.get( + "validator_count", + lean_network_params["num_validator_keys_per_node"], + ), + "is_aggregator": participant.get("is_aggregator", False), + "extra_params": participant.get("lean_extra_params", []), + "extra_env_vars": participant.get("lean_extra_env_vars", {}), + "extra_labels": participant.get("lean_extra_labels", {}), + "log_level": participant.get("lean_log_level", ""), + "min_cpu": participant.get("lean_min_cpu", 0), + "max_cpu": participant.get("lean_max_cpu", 0), + "min_mem": participant.get("lean_min_mem", 0), + "max_mem": participant.get("lean_max_mem", 0), + "node_selectors": participant.get("node_selectors", {}), + "tolerations": participant.get("tolerations", []), + "prometheus_config": participant.get( + "prometheus_config", + { + "scrape_interval": "15s", + "labels": {}, + }, + ), + # Optional Engine API pairing — populated by main.star + # when the participant has a non-`none` el_type. + # None for participants with `el_type: none` (Lean-only). + "_el_context": participant.get("_el_context"), + } + ) + + node_names = [n["node_name"] for n in expanded] + keys_artifact = p2p_keys.generate_node_keys(plan, node_names) + + # Compute the total validator count up front so we can generate the + # hash-sig keys before any service is added. Hash-sig keys don't depend + # on the node IP, so we can mount them directly at initialize() time + # and avoid having to re-mount any artifact later (Kurtosis rejects + # two add_service calls with the same name). + total_validators = 0 + for node in expanded: + total_validators += node["validator_count"] + hash_sig_artifact = lean_genesis.generate_hash_sig_keys( + plan, + lean_network_params, + total_validators, + ) + + # Stash both artifacts on each node record so per-client launchers + # have access during initialize(). + for node in expanded: + node["_p2p_keys_artifact"] = keys_artifact + node["_hash_sig_artifact"] = hash_sig_artifact + + # Phase 1: initialise placeholder services so Kurtosis assigns IPs. + services = [] + for node in expanded: + launcher = _launcher_for(node["lean_type"]) + service = launcher.initialize( + plan, + node, + keys_artifact, + hash_sig_artifact, + ) + services.append((node, service)) + + # Phase 2: render the validator-config.yaml and run the genesis tool now + # that every service has an IP. Note that we do NOT pass per-node + # privkey values from Starlark — `plan.run_sh(...).output` is a runtime + # future, so individual keys can't be looked up here. The genesis + # pipeline reads `.key` directly from the keys artifact inside + # its own shell. + services_meta = [] + for node, service in services: + services_meta.append( + { + "name": node["node_name"], + "ip_address": service.ip_address, + "quic_port": constants.LEAN_QUIC_PORT_NUM, + "metrics_port": constants.LEAN_METRICS_PORT_NUM, + "api_port": constants.LEAN_API_PORT_NUM, + "validator_count": node["validator_count"], + "is_aggregator": node["is_aggregator"], + } + ) + + genesis = lean_genesis.generate( + plan, + services_meta, + lean_network_params, + keys_artifact, + hash_sig_artifact, + ) + + # Phase 2.5: for nodes with a paired EL, query the EL's genesis + # block hash. ethlambda's `--execution-genesis-block-hash` flag seeds + # `state.latest_execution_payload_header.block_hash` so the very first + # `engine_forkchoiceUpdatedV3` carries a head the EL recognizes — + # without it the EL replies SYNCING forever. The hash is the same for + # every node sharing the same EL chain, but we still query per-EL + # because each Lean node points at a different EL service. + for node in expanded: + el_context = node["_el_context"] + if el_context == None: + node["_el_genesis_block_hash"] = None + continue + # `eth_getBlockByNumber 0x0` returns the genesis header; `.hash` + # is the 0x-prefixed 32-byte hash ethlambda expects. We strip + # quotes/newlines for clean substitution into the CLI flag. + rpc_url = "http://{0}:{1}".format( + el_context.ip_addr, el_context.rpc_port_num + ) + query = plan.run_sh( + run='set -eu; out=$(curl -sf -X POST -H "Content-Type: application/json" ' + + '--data \'{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["0x0",false],"id":1}\' ' + + '{0}); echo "$out" | sed -E \'s/.*"hash":"(0x[0-9a-fA-F]+)".*/\\1/\''.format( + rpc_url + ), + description="Querying EL genesis block hash for {0}".format( + node["node_name"] + ), + wait="5m", + ) + node["_el_genesis_block_hash"] = query.output + + # Phase 3: hand off to each per-client launcher to mount the genesis bundle + # and start the real binary. ethlambda accepts optional EL params; the + # other Lean launchers ignore them. + contexts = [] + for node, service in services: + launcher = _launcher_for(node["lean_type"]) + if ( + node["lean_type"] == constants.LEAN_TYPE.ethlambda + and node["_el_context"] != None + ): + ctx = launcher.start( + plan, + node, + service, + genesis.genesis_artifact, + genesis.hash_sig_artifact, + el_context=node["_el_context"], + jwt_file=jwt_file, + el_genesis_block_hash=node["_el_genesis_block_hash"], + ) + else: + ctx = launcher.start( + plan, + node, + service, + genesis.genesis_artifact, + genesis.hash_sig_artifact, + ) + contexts.append(ctx) + + plan.print( + "Lean pipeline ready: {0} nodes, GENESIS_TIME={1}".format( + len(contexts), + genesis.genesis_time, + ) + ) + + # Phase 4: launch Prometheus + Grafana scraping every Lean node's + # /metrics endpoint. Enabled by default; set + # lean_network_params.metrics_enabled: false to skip. + if lean_network_params.get("metrics_enabled", True): + metrics_launcher.launch(plan, contexts) + + return contexts diff --git a/src/lean/lean_shared.star b/src/lean/lean_shared.star new file mode 100644 index 000000000..ce2c1885e --- /dev/null +++ b/src/lean/lean_shared.star @@ -0,0 +1,86 @@ +""" +Shared helpers for Lean client launchers. + +Every Lean client speaks the same wire protocols (libp2p QUIC + JSON REST + +Prometheus). Keeping the port specs and mountpoint contract in one module +lets per-client launchers stay focused on CLI translation. +""" + +constants = import_module("../package_io/constants.star") + + +def lean_port_specs(): + """Default port spec triple for any Lean client. + + QUIC is the only P2P protocol (no TCP discovery); the API and metrics + endpoints are plain HTTP. Each Lean client maps these to its own CLI + flag names (`--gossipsub-port`, `--http-port`, etc.) inside its + launcher. + """ + return { + constants.LEAN_QUIC_PORT_ID: PortSpec( + number=constants.LEAN_QUIC_PORT_NUM, + transport_protocol="UDP", + application_protocol="quic", + wait=None, + ), + constants.LEAN_API_PORT_ID: PortSpec( + number=constants.LEAN_API_PORT_NUM, + transport_protocol="TCP", + application_protocol="http", + wait=None, + ), + constants.LEAN_METRICS_PORT_ID: PortSpec( + number=constants.LEAN_METRICS_PORT_NUM, + transport_protocol="TCP", + application_protocol="http", + wait=None, + ), + } + + +def lean_log_file_path(service_name): + """Path inside the container where the Lean client's logs are tailed. + + Stored under /var/log so a single `tail -f` keeps the Kurtosis service + "alive" while we initialise it; the client itself runs as a backgrounded + `nohup`. Matches the pattern ReamLabs/pq-devnet-package established. + """ + return "/var/log/{0}.log".format(service_name) + + +def lean_tail_logs_cmd(service_name): + """Initial container command — touches and tails the log file. + + Holds the service open until the actual client binary is started by a + follow-up `plan.exec`. Without this, Kurtosis would mark the service + failed before we got a chance to mount the genesis bundle. + """ + log_file = lean_log_file_path(service_name) + return ["/bin/sh", "-c", "touch {0} && tail -f {0}".format(log_file)] + + +def common_cfg_kwargs(node): + """ServiceConfig kwargs shared between per-client initialize() and start(). + + Kurtosis rejects memory/cpu values of 0 (it expects "unset" via the + *absence* of the kwarg, not via a 0 sentinel). We omit those keys here + when the participant didn't set them. Per-client launchers add image, + cmd, entrypoint, and files on top. + """ + kwargs = { + "ports": lean_port_specs(), + "env_vars": node["extra_env_vars"], + "labels": node["extra_labels"], + "node_selectors": node["node_selectors"], + "tolerations": node["tolerations"], + } + if node["min_cpu"] > 0: + kwargs["min_cpu"] = node["min_cpu"] + if node["max_cpu"] > 0: + kwargs["max_cpu"] = node["max_cpu"] + if node["min_mem"] > 0: + kwargs["min_memory"] = node["min_mem"] + if node["max_mem"] > 0: + kwargs["max_memory"] = node["max_mem"] + return kwargs diff --git a/src/lean/lighthouse/lighthouse_launcher.star b/src/lean/lighthouse/lighthouse_launcher.star new file mode 100644 index 000000000..ce1b204c4 --- /dev/null +++ b/src/lean/lighthouse/lighthouse_launcher.star @@ -0,0 +1,144 @@ +""" +lighthouse (lean) launcher. + +Translates the Lean pipeline's per-node record into the Lean lighthouse +fork's CLI surface from `client-cmds/lighthouse-cmd.sh` in +blockblaz/lean-quickstart, restricted to the flags the published +`hopinheimer/lighthouse:latest` image actually accepts. + +NOTE: the image's `lighthouse lean_node` subcommand does NOT support +`--api-port` or `--is-aggregator`. Lighthouse will run as a non-aggregator +peer with only the metrics endpoint exposed; setting `is_aggregator: true` +on this participant is silently ignored for this client. +""" + +constants = import_module("../../package_io/constants.star") +lean_shared = import_module("../lean_shared.star") +lean_context = import_module("../lean_context.star") + +ENTRYPOINT = "/usr/local/bin/lighthouse" +GENESIS_MOUNT = constants.LEAN_GENESIS_MOUNTPOINT_ON_CLIENTS +HASH_SIG_MOUNT = GENESIS_MOUNT + "/hash-sig-keys" +DATA_DIR = "/data" +NODE_KEY_MOUNT = constants.LEAN_NODE_KEY_MOUNTPOINT_ON_CLIENTS + +GENESIS_TEXT_FILES = [ + "config.yaml", + "annotated_validators.yaml", + "nodes.yaml", + "validator-config.yaml", + "validators.yaml", + "genesis.json", +] + + +def initialize(plan, node, p2p_keys_artifact, hash_sig_artifact): + cfg_kwargs = lean_shared.common_cfg_kwargs(node) + cfg_kwargs.update( + { + "image": node["image"], + "entrypoint": ["/bin/sh", "-c"], + "cmd": lean_shared.lean_tail_logs_cmd(node["service_name"])[2:], + "files": { + NODE_KEY_MOUNT: p2p_keys_artifact, + HASH_SIG_MOUNT: hash_sig_artifact, + }, + } + ) + return plan.add_service(node["service_name"], ServiceConfig(**cfg_kwargs)) + + +def start(plan, node, service, genesis_artifact, hash_sig_artifact): + service_name = service.name + log_file = lean_shared.lean_log_file_path(service_name) + + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=["/bin/sh", "-c", "mkdir -p {0}".format(GENESIS_MOUNT)], + ), + description="Preparing genesis mount on {0}".format(service_name), + ) + for filename in GENESIS_TEXT_FILES: + read = plan.run_sh( + run="cat /src/{0}".format(filename), + files={"/src": genesis_artifact}, + description="Reading {0} for {1}".format(filename, service_name), + ) + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=[ + "/bin/sh", + "-c", + "cat > {0}/{1} <<'LIGHTHOUSE_EOF'\n{2}\nLIGHTHOUSE_EOF".format( + GENESIS_MOUNT, + filename, + read.output, + ), + ], + ), + description="Staging {0} into {1}".format(filename, service_name), + ) + + cmd_parts = [ + ENTRYPOINT, + "--datadir", + DATA_DIR, + "lean_node", + "--config", + "{0}/config.yaml".format(GENESIS_MOUNT), + "--validators", + "{0}/validator-config.yaml".format(GENESIS_MOUNT), + "--nodes", + "{0}/nodes.yaml".format(GENESIS_MOUNT), + "--node-id", + node["node_name"], + "--private-key", + "{0}/{1}.key".format(NODE_KEY_MOUNT, node["node_name"]), + "--genesis-json", + "{0}/genesis.json".format(GENESIS_MOUNT), + "--socket-port", + str(constants.LEAN_QUIC_PORT_NUM), + "--metrics", + "--metrics-address", + "0.0.0.0", + "--metrics-port", + str(constants.LEAN_METRICS_PORT_NUM), + ] + for extra in node["extra_params"]: + cmd_parts.append(extra) + + nohup_cmd = "setsid -f sh -c \"exec {0}\" < /dev/null >> {1} 2>&1".format( + " ".join(cmd_parts), + log_file, + ) + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=["/bin/sh", "-c", nohup_cmd], + ), + description="Starting lighthouse binary on {0}".format(service_name), + ) + + api_url = "http://{0}:{1}".format(service.ip_address, constants.LEAN_API_PORT_NUM) + metrics_url = "http://{0}:{1}/metrics".format( + service.ip_address, constants.LEAN_METRICS_PORT_NUM + ) + + return lean_context.new_lean_context( + client_name=constants.LEAN_TYPE.lighthouse, + service_name=service_name, + ip_address=service.ip_address, + quic_port=constants.LEAN_QUIC_PORT_NUM, + api_port=constants.LEAN_API_PORT_NUM, + metrics_port=constants.LEAN_METRICS_PORT_NUM, + api_url=api_url, + metrics_url=metrics_url, + metrics_info={ + "name": service_name, + "url": metrics_url, + "path": "/metrics", + "config": node["prometheus_config"], + }, + ) diff --git a/src/lean/metrics/grafana/dashboards/client-dashboard.json b/src/lean/metrics/grafana/dashboards/client-dashboard.json new file mode 100644 index 000000000..9c4aad3d8 --- /dev/null +++ b/src/lean/metrics/grafana/dashboards/client-dashboard.json @@ -0,0 +1,6803 @@ +{ + "__inputs": [ + { + "name": "datasource", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "12.3.2" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 16, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dateTimeAsIso" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 76, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 20 + }, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by(network) (lean_node_start_time_seconds{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"} * 1000)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Latest start time", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 103, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(network) (lean_validators_count{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Total number of validators", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 40, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by(network) (lean_attestation_committee_count{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Number of attestation committees", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of validators attached to each node", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 51, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum (lean_validators_count)", + "hide": false, + "instant": true, + "legendFormat": "Total", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (network, job, instance) (lean_validators_count{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "interval": "", + "legendFormat": "{{job}}", + "range": false, + "refId": "B", + "useBackend": false + } + ], + "title": "Validators list", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-green", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 89, + "options": { + "cellHeight": "sm", + "showHeader": false + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by(network,job,instance) (lean_is_aggregator{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}) > 0", + "instant": true, + "legendFormat": "{{job}}", + "range": false, + "refId": "A" + } + ], + "title": "Aggregators", + "transformations": [ + { + "id": "labelsToFields", + "options": {} + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "job" + ] + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "name" + }, + "properties": [ + { + "id": "custom.width", + "value": 112 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.width", + "value": 116 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 75, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "last_over_time(lean_node_info{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[1m])", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Node info", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "keepLabels": [ + "name", + "version", + "job" + ] + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "job", + "version" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "job": "Node", + "name": "Client", + "version": "Version" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 5 + }, + "id": 88, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by(network) (lean_latest_finalized_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Latest finalized slot", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 5 + }, + "id": 86, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by(network) (lean_latest_justified_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Latest justified slot", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 5 + }, + "id": 87, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by(network) (lean_head_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Head slot", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 6, + "x": 0, + "y": 9 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (network, job, instance)(lean_latest_finalized_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": " Latest finalized slot", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 6, + "x": 6, + "y": 9 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": " sum by (network, job, instance) (lean_latest_justified_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Latest justified slot", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 6, + "x": 12, + "y": 9 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": " sum by (network, job, instance) (lean_head_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Head slot", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 6, + "x": 18, + "y": 9 + }, + "id": 66, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": " sum by (network, job, instance) (lean_current_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Current slot", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of processed slots in state transition function", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "changes(lean_node_start_time_seconds{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[1m])", + "instant": false, + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Start time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 90, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (network, job, instance) (lean_attestation_committee_subnet{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Attestation committee subnet", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": " sum by (network, job, instance) (lean_connected_peers{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Connected peers per node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "fillOpacity": 0, + "spanNulls": false, + "showPoints": "never", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [ + { + "options": { + "0": { + "text": "idle" + }, + "1": { + "text": "syncing" + }, + "2": { + "text": "synced" + } + }, + "type": "value" + } + ], + "min": 0, + "max": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 98, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max by (network, job, instance) (\n (lean_node_sync_status{status=\"synced\", network=~\"$network\", job=~\"$job\", instance=~\"$instance\"} == 1) * 2\n or\n (lean_node_sync_status{status=\"syncing\", network=~\"$network\", job=~\"$job\", instance=~\"$instance\"} == 1) * 1\n or\n (lean_node_sync_status{status=\"idle\", network=~\"$network\", job=~\"$job\", instance=~\"$instance\"} == 1) * 0\n)", + "instant": false, + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Node sync status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of processed slots in state transition function", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg_over_time(\n sum by (network, job, instance) (increase(lean_finalizations_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\", result=\"success\"}[1m]))[5m:]\n)", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Finalizations - Success", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg_over_time(\n sum by (network, job, instance) (increase(lean_finalizations_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\", result=\"error\"}[1m]))[5m:]\n)", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Finalizations - Errors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 57, + "panels": [], + "title": "Finalization/Justification Delay", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 48 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg_over_time(lean_head_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[5m]) - avg_over_time(lean_latest_finalized_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[5m])", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Head - Finalized delay (slots)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 48 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg_over_time(lean_head_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[5m]) - avg_over_time(lean_latest_justified_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[5m])", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Head - Justified delay (slots)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 48 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg_over_time(lean_latest_justified_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[5m]) - avg_over_time(lean_latest_finalized_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[5m])", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Justified - Finalized delay (slots)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 53, + "panels": [], + "title": "Peers", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": " sum by (network, job, instance) (lean_connected_peers{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Connected peers per node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": " sum by (network, job, instance, client) (lean_connected_peers{network=~\"$network\", job=~\"$job\", instance=~\"$instance\", client!=\"\"})", + "instant": false, + "interval": "", + "legendFormat": "{{job}} - {{client}}", + "range": true, + "refId": "A" + } + ], + "title": "Connected peers per node (detailed)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 68 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (network, job, instance) (\n increase(lean_peer_connection_events_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )", + "interval": "", + "legendFormat": "{{job}} {{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Peer connection events", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 68 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (network, job, instance) (\n increase(lean_peer_disconnection_events_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )", + "interval": "", + "legendFormat": "{{job}} {{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Peer disconnection events", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 77 + }, + "id": 99, + "panels": [], + "title": "Gossip messages", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 78 + }, + "id": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \n rate(lean_gossip_block_size_bytes_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Bytes size of a gossip block message", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 78 + }, + "id": 101, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \n rate(lean_gossip_attestation_size_bytes_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Bytes size of a gossip attestation message", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 78 + }, + "id": 102, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \n rate(lean_gossip_aggregation_size_bytes_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Bytes size of a gossip aggregated attestation message", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 86 + }, + "id": 45, + "panels": [], + "title": "PQ Signatures", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of individual attestation signatures", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 87 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(\n sum by (network, job, instance) (\n increase(lean_pq_sig_attestation_signatures_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )[5m:]\n)", + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Total number of attestation signatures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of valid individual attestation signatures", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 87 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(\n sum by (network, job, instance) (\n increase(lean_pq_sig_attestation_signatures_valid_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )[5m:]\n)", + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Total number of valid attestation signatures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of invalid individual attestation signatures", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 87 + }, + "id": 65, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(\n sum by (network, job, instance) (\n increase(lean_pq_sig_attestation_signatures_invalid_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )[5m:]\n)", + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Total number of invalid attestation signatures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 95 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.99, rate(lean_pq_sig_attestation_signing_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Time taken to sign an attestation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 95 + }, + "id": 47, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.99, rate(lean_pq_sig_attestation_verification_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Time taken to verify an attestation signature", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 103 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(\n sum by (network, job, instance) (\n increase(lean_pq_sig_aggregated_signatures_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )[5m:]\n)", + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Total number of aggregated signatures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 103 + }, + "id": 61, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(\n sum by (network, job, instance) (\n increase(lean_pq_sig_attestations_in_aggregated_signatures_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )[5m:]\n)", + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Total number of attestations included into aggregated signatures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 111 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(\n sum by (network, job, instance) (\n increase(lean_pq_sig_aggregated_signatures_valid_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )[5m:]\n)", + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Total number of valid aggregated signatures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 111 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(\n sum by (network, job, instance) (\n increase(lean_pq_sig_aggregated_signatures_invalid_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )[5m:]\n)", + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Total number of invalid aggregated signatures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 119 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.99, rate(lean_pq_sig_aggregated_signatures_building_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Time taken to build an aggregated signature", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 119 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.99, rate(lean_pq_sig_aggregated_signatures_verification_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Time taken to verify an aggregated signature", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 127 + }, + "id": 91, + "panels": [], + "title": "Block production", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 128 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \n rate(lean_block_aggregated_payloads_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Number of aggregated payloads", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 128 + }, + "id": 93, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \n rate(lean_block_building_payload_aggregation_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Time taken to build aggregated payloads", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 136 + }, + "id": 80, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": " avg_over_time (\n sum by (network, job, instance) (\n (lean_block_building_success_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})\n )[2m:]\n)", + "instant": false, + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Successful block builds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 136 + }, + "id": 97, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": " avg_over_time (\n sum by (network, job, instance) (\n (lean_block_building_failures_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})\n )[2m:]\n)", + "instant": false, + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Failed block builds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 136 + }, + "id": 94, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \n rate(lean_block_building_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Time taken to build a block", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 144 + }, + "id": 17, + "panels": [], + "title": "Fork-Choice", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Time taken to process block in fork-choice", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 145 + }, + "id": 92, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \n rate(lean_fork_choice_block_processing_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Block processing time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Time taken to process block in fork-choice", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 145 + }, + "id": 85, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \n rate(lean_committee_signatures_aggregation_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Time taken to aggregate committee signatures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 153 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (network, job, instance) (\n increase(lean_fork_choice_reorgs_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "{{job}} {{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Total number of fork choice reorgs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 153 + }, + "id": 69, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, rate(lean_fork_choice_reorg_depth_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{job}} {{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Depth of fork choice reorgs (in blocks)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 161 + }, + "id": 96, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": " avg_over_time (\n sum by (network, job, instance) (\n (lean_gossip_signatures{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})\n )[2m:]\n)", + "instant": false, + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Number of gossip signatures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 161 + }, + "id": 81, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(\n sum by (network, job, instance) (\n (lean_latest_new_aggregated_payloads{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})\n )[2m:]\n)", + "instant": false, + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Number of new aggregated payloads", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 161 + }, + "id": 82, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(\n sum by (network, job, instance) (\n (lean_latest_known_aggregated_payloads{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})\n )[1m:]\n)", + "instant": false, + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Number of known aggregated payloads", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 169 + }, + "id": 8, + "panels": [], + "title": "Attestations", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 170 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg_over_time(\n sum by (network, job, instance) (\n increase(lean_attestations_valid_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )[5m:]\n)", + "interval": "", + "legendFormat": "{{job}} {{source}}", + "range": true, + "refId": "A" + } + ], + "title": "FC Valid attestations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 170 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg_over_time(\n sum by (network, job, instance) (increase(lean_attestations_invalid_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))[5m:]\n)", + "legendFormat": "{{job}} {{source}}", + "range": true, + "refId": "A" + } + ], + "title": "FC Invalid attestations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 170 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, rate(lean_attestation_validation_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "FC Attestations validation time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 178 + }, + "id": 67, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": " sum by (network, job, instance) (lean_safe_target_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Safe target slot", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of attestations processed in state transition function", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 178 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(\n sum by (network, job, instance) (increase(lean_state_transition_attestations_processed_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))[5m:]\n)", + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "STF Processed attestations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Time taken to process attestations in state transition function", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 178 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99,\n rate(lean_state_transition_attestations_processing_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "STF Attestations processing time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 186 + }, + "id": 83, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, rate(lean_attestations_production_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{job}} {{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Time taken to produce attestation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 186 + }, + "id": 84, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, rate(lean_committee_signatures_aggregation_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Time taken to aggregate committee signatures", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 194 + }, + "id": 21, + "panels": [], + "title": "State Transition", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Time taken to process state transition function", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 195 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \n rate(lean_state_transition_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "State transition time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Time taken to process block in state transition function", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 195 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \n rate(lean_state_transition_block_processing_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Block processing time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of processed slots in state transition function", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 203 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg_over_time(\n sum by (network, job, instance) (increase(lean_state_transition_slots_processed_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))[5m:]\n)", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Processed slots", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Time taken to process slots in state transition function", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 203 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99,\n rate(lean_state_transition_slots_processing_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Slots processing time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of processed slots in state transition function", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 211 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg_over_time(\n sum by (network, job, instance) (increase(lean_finalizations_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\", result=\"success\"}[1m]))[5m:]\n)", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Finalizations - Success", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 211 + }, + "id": 71, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg_over_time(\n sum by (network, job, instance) (increase(lean_finalizations_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\", result=\"error\"}[1m]))[5m:]\n)", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Finalizations - Errors", + "type": "timeseries" + } + ], + "preload": false, + "schemaVersion": 42, + "tags": [ + "interop", + "Client" + ], + "templating": { + "list": [ + { + "allowCustomValue": false, + "current": {}, + "definition": "label_values(network)", + "label": "Network", + "name": "network", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(network)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allowCustomValue": false, + "current": {}, + "definition": "label_values(lean_node_info{network=~\"$network\"}, job)", + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(lean_node_info{network=~\"$network\"}, job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": ".*ethlambda.*|.*gean.*|.*grandine.*|.*lantern.*|.*lighthouse.*|.*qlean.*|.*ream.*|.*zeam.*", + "type": "query" + }, + { + "allowCustomValue": false, + "current": {}, + "definition": "label_values(lean_node_info{network=~\"$network\", job=~\"$job\"}, instance)", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(lean_node_info{network=~\"$network\", job=~\"$job\"}, instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allowCustomValue": false, + "current": { + "text": "", + "value": "${datasource}", + "selected": true + }, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Lean Ethereum Clients Dashboard", + "uid": "lean-ethereum-clients-dashboard", + "version": 25, + "weekStart": "", + "id": null +} diff --git a/src/lean/metrics/metrics_launcher.star b/src/lean/metrics/metrics_launcher.star new file mode 100644 index 000000000..6546a4b9b --- /dev/null +++ b/src/lean/metrics/metrics_launcher.star @@ -0,0 +1,217 @@ +""" +Prometheus + Grafana for the Lean Ethereum devnet. + +Mirrors the docker-compose-metrics.yaml stack shipped in +blockblaz/lean-quickstart (prom v3.8.0, grafana v12.3.2, anonymous admin +login). Inside the Kurtosis enclave we scrape by service DNS name + port +instead of host.docker.internal:port, so no `--add-host` hackery is needed. + +Layout: + * One Prometheus service named "lean-prometheus" scraping every + lean--:5054/metrics target plus its own /metrics. + * One Grafana service named "lean-grafana" provisioning the Prometheus + datasource and the upstream Lean client dashboard. + +The whole thing is enabled by default whenever any participant has a +Lean cl_type; set `lean_network_params.metrics_enabled: false` to skip it. +""" + +constants = import_module("../../package_io/constants.star") + +PROMETHEUS_IMAGE = "prom/prometheus:v3.8.0" +GRAFANA_IMAGE = "grafana/grafana:12.3.2" +PROMETHEUS_SERVICE = "lean-prometheus" +GRAFANA_SERVICE = "lean-grafana" +PROMETHEUS_PORT = 9090 +GRAFANA_PORT = 3000 + +DASHBOARD_REL = "src/lean/metrics/grafana/dashboards/client-dashboard.json" + + +def launch(plan, lean_contexts): + """Spin up Prometheus + Grafana for the given Lean nodes. + + Args: + plan: Kurtosis plan. + lean_contexts: list of `lean_context.new_lean_context` structs. + Empty / None disables the stack. + """ + if not lean_contexts: + return None + + prometheus_artifact = _render_prometheus_config(plan, lean_contexts) + grafana_provisioning = _render_grafana_provisioning(plan) + grafana_dashboards = plan.upload_files( + src="/" + DASHBOARD_REL, + name="lean-grafana-dashboards", + ) + + # Pin host-side ports via public_ports so dashboards have stable URLs + # across re-runs and so the operator can reach them via the host's + # public hostname (e.g. http://my-host:3000) without an SSH tunnel. + # Kurtosis's default `ports={}` publishes on a random host port bound + # to 127.0.0.1; `public_ports={}` pins the host-side port and Docker + # publishes it on 0.0.0.0 (the default `-p` behaviour). + prometheus = plan.add_service( + name=PROMETHEUS_SERVICE, + config=ServiceConfig( + image=PROMETHEUS_IMAGE, + cmd=[ + "--config.file=/etc/prometheus/prometheus.yml", + "--storage.tsdb.path=/prometheus", + "--storage.tsdb.retention.time=30d", + "--web.enable-lifecycle", + ], + ports={ + "http": PortSpec( + number=PROMETHEUS_PORT, + transport_protocol="TCP", + application_protocol="http", + wait=None, + ), + }, + public_ports={ + "http": PortSpec( + number=PROMETHEUS_PORT, + transport_protocol="TCP", + application_protocol="http", + wait=None, + ), + }, + files={"/etc/prometheus": prometheus_artifact}, + ), + ) + + grafana = plan.add_service( + name=GRAFANA_SERVICE, + config=ServiceConfig( + image=GRAFANA_IMAGE, + env_vars={ + "GF_SECURITY_ADMIN_USER": "admin", + "GF_SECURITY_ADMIN_PASSWORD": "admin", + "GF_USERS_ALLOW_SIGN_UP": "false", + "GF_AUTH_ANONYMOUS_ENABLED": "true", + "GF_AUTH_ANONYMOUS_ORG_ROLE": "Admin", + "GF_AUTH_DISABLE_LOGIN_FORM": "true", + }, + ports={ + "http": PortSpec( + number=GRAFANA_PORT, + transport_protocol="TCP", + application_protocol="http", + wait=None, + ), + }, + public_ports={ + "http": PortSpec( + number=GRAFANA_PORT, + transport_protocol="TCP", + application_protocol="http", + wait=None, + ), + }, + files={ + "/etc/grafana/provisioning": grafana_provisioning, + "/var/lib/grafana/dashboards": grafana_dashboards, + }, + ), + ) + + plan.print( + "Lean metrics ready: prometheus={0}:{1}, grafana={2}:{3}".format( + prometheus.ip_address, + PROMETHEUS_PORT, + grafana.ip_address, + GRAFANA_PORT, + ) + ) + return struct( + prometheus=prometheus, + grafana=grafana, + ) + + +def _render_prometheus_config(plan, lean_contexts): + """Render prometheus.yml with one scrape target per Lean node.""" + template = """# Auto-generated by src/lean/metrics/metrics_launcher.star. +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + monitor: "lean-devnet-metrics" + +scrape_configs: +{{- range .Targets}} + - job_name: "{{.Name}}" + static_configs: + - targets: ["{{.Target}}"] + labels: + client: "{{.Name}}" + instance: "kurtosis" +{{end}} + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] +""" + targets = [] + for ctx in lean_contexts: + targets.append( + { + "Name": ctx.service_name, + # Kurtosis services resolve by service name within the + # enclave network; ports are the internal port (not the + # published host port). All Lean clients standardise on + # 5054 for metrics. + "Target": "{0}:{1}".format(ctx.service_name, ctx.metrics_port), + } + ) + return plan.render_templates( + config={ + "prometheus.yml": struct(template=template, data={"Targets": targets}), + }, + name="lean-prometheus-config", + description="Rendering Lean prometheus.yml", + ) + + +def _render_grafana_provisioning(plan): + """Render Grafana provisioning YAML files (datasource + dashboards).""" + return plan.render_templates( + config={ + "datasources/prometheus.yml": struct( + template="""apiVersion: 1 + +datasources: + - name: prometheus + type: prometheus + uid: P1809F7CD0C75ACF3 + access: proxy + url: http://{0}:{1} + isDefault: true + editable: true +""".format( + PROMETHEUS_SERVICE, PROMETHEUS_PORT + ), + data={}, + ), + "dashboards/dashboards.yml": struct( + template="""apiVersion: 1 + +providers: + - name: "Lean Ethereum Dashboards" + orgId: 1 + folder: "" + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: true +""", + data={}, + ), + }, + name="lean-grafana-provisioning", + description="Rendering Lean Grafana provisioning", + ) diff --git a/src/lean/qlean/qlean_launcher.star b/src/lean/qlean/qlean_launcher.star new file mode 100644 index 000000000..09b37d414 --- /dev/null +++ b/src/lean/qlean/qlean_launcher.star @@ -0,0 +1,142 @@ +""" +qlean launcher. + +Translates the Lean pipeline's per-node record into qlean's CLI surface +from `client-cmds/qlean-cmd.sh` in blockblaz/lean-quickstart: + + qlean \ + --genesis-dir \ + --data-dir \ + --node-id --node-key \ + --listen-addr /ip4/0.0.0.0/udp//quic-v1 \ + --metrics-host 0.0.0.0 --metrics-port \ + --api-host 0.0.0.0 --api-port +""" + +constants = import_module("../../package_io/constants.star") +lean_shared = import_module("../lean_shared.star") +lean_context = import_module("../lean_context.star") + +ENTRYPOINT = "/opt/qlean/bin/qlean" +GENESIS_MOUNT = constants.LEAN_GENESIS_MOUNTPOINT_ON_CLIENTS +HASH_SIG_MOUNT = GENESIS_MOUNT + "/hash-sig-keys" +DATA_DIR = "/data" +NODE_KEY_MOUNT = constants.LEAN_NODE_KEY_MOUNTPOINT_ON_CLIENTS + +GENESIS_TEXT_FILES = [ + "config.yaml", + "annotated_validators.yaml", + "nodes.yaml", + "validator-config.yaml", + "validators.yaml", +] + + +def initialize(plan, node, p2p_keys_artifact, hash_sig_artifact): + cfg_kwargs = lean_shared.common_cfg_kwargs(node) + cfg_kwargs.update( + { + "image": node["image"], + "entrypoint": ["/bin/sh", "-c"], + "cmd": lean_shared.lean_tail_logs_cmd(node["service_name"])[2:], + "files": { + NODE_KEY_MOUNT: p2p_keys_artifact, + HASH_SIG_MOUNT: hash_sig_artifact, + }, + } + ) + return plan.add_service(node["service_name"], ServiceConfig(**cfg_kwargs)) + + +def start(plan, node, service, genesis_artifact, hash_sig_artifact): + service_name = service.name + log_file = lean_shared.lean_log_file_path(service_name) + + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=["/bin/sh", "-c", "mkdir -p {0}".format(GENESIS_MOUNT)], + ), + description="Preparing genesis mount on {0}".format(service_name), + ) + for filename in GENESIS_TEXT_FILES: + read = plan.run_sh( + run="cat /src/{0}".format(filename), + files={"/src": genesis_artifact}, + description="Reading {0} for {1}".format(filename, service_name), + ) + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=[ + "/bin/sh", + "-c", + "cat > {0}/{1} <<'QLEAN_EOF'\n{2}\nQLEAN_EOF".format( + GENESIS_MOUNT, + filename, + read.output, + ), + ], + ), + description="Staging {0} into {1}".format(filename, service_name), + ) + + cmd_parts = [ + ENTRYPOINT, + "--genesis-dir", + GENESIS_MOUNT, + "--data-dir", + DATA_DIR, + "--node-id", + node["node_name"], + "--node-key", + "{0}/{1}.key".format(NODE_KEY_MOUNT, node["node_name"]), + "--listen-addr", + "/ip4/0.0.0.0/udp/{0}/quic-v1".format(constants.LEAN_QUIC_PORT_NUM), + "--metrics-host", + "0.0.0.0", + "--metrics-port", + str(constants.LEAN_METRICS_PORT_NUM), + "--api-host", + "0.0.0.0", + "--api-port", + str(constants.LEAN_API_PORT_NUM), + ] + if node["is_aggregator"]: + cmd_parts.append("--is-aggregator") + for extra in node["extra_params"]: + cmd_parts.append(extra) + + nohup_cmd = "setsid -f sh -c \"exec {0}\" < /dev/null >> {1} 2>&1".format( + " ".join(cmd_parts), + log_file, + ) + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=["/bin/sh", "-c", nohup_cmd], + ), + description="Starting qlean binary on {0}".format(service_name), + ) + + api_url = "http://{0}:{1}".format(service.ip_address, constants.LEAN_API_PORT_NUM) + metrics_url = "http://{0}:{1}/metrics".format( + service.ip_address, constants.LEAN_METRICS_PORT_NUM + ) + + return lean_context.new_lean_context( + client_name=constants.LEAN_TYPE.qlean, + service_name=service_name, + ip_address=service.ip_address, + quic_port=constants.LEAN_QUIC_PORT_NUM, + api_port=constants.LEAN_API_PORT_NUM, + metrics_port=constants.LEAN_METRICS_PORT_NUM, + api_url=api_url, + metrics_url=metrics_url, + metrics_info={ + "name": service_name, + "url": metrics_url, + "path": "/metrics", + "config": node["prometheus_config"], + }, + ) diff --git a/src/lean/ream/ream_launcher.star b/src/lean/ream/ream_launcher.star new file mode 100644 index 000000000..a49506c6e --- /dev/null +++ b/src/lean/ream/ream_launcher.star @@ -0,0 +1,165 @@ +""" +ream launcher. + +Mirrors the ethlambda launcher shape (placeholder service -> genesis pipeline +-> re-issue with full mounts) but translates to ream's CLI surface as +documented in `client-cmds/ream-cmd.sh` in blockblaz/lean-quickstart: + + ream --data-dir lean_node \ + --network \ + --validator-registry-path \ + --bootnodes \ + --node-id --node-key \ + --socket-port \ + --metrics --metrics-address 0.0.0.0 --metrics-port \ + --http-address 0.0.0.0 --http-port + +NOTE: ream's `lean_node` subcommand doesn't (yet) consume a hash-sig-keys +directory in its public CLI - it derives its validator set from +`annotated_validators.yaml` + GENESIS_VALIDATORS in config.yaml. The genesis +artifact we mount carries both, so ream needs no extra plumbing. +""" + +constants = import_module("../../package_io/constants.star") +lean_shared = import_module("../lean_shared.star") +lean_context = import_module("../lean_context.star") + +ENTRYPOINT = "/usr/local/bin/ream" +GENESIS_MOUNT = constants.LEAN_GENESIS_MOUNTPOINT_ON_CLIENTS +HASH_SIG_MOUNT = GENESIS_MOUNT + "/hash-sig-keys" +DATA_DIR = "/data" +NODE_KEY_MOUNT = constants.LEAN_NODE_KEY_MOUNTPOINT_ON_CLIENTS + + +GENESIS_TEXT_FILES = [ + "config.yaml", + "annotated_validators.yaml", + "nodes.yaml", + "validator-config.yaml", +] + + +def initialize(plan, node, p2p_keys_artifact, hash_sig_artifact): + cfg_kwargs = lean_shared.common_cfg_kwargs(node) + cfg_kwargs.update( + { + "image": node["image"], + "entrypoint": ["/bin/sh", "-c"], + "cmd": lean_shared.lean_tail_logs_cmd(node["service_name"])[2:], + "files": { + NODE_KEY_MOUNT: p2p_keys_artifact, + HASH_SIG_MOUNT: hash_sig_artifact, + }, + } + ) + return plan.add_service(node["service_name"], ServiceConfig(**cfg_kwargs)) + + +def start(plan, node, service, genesis_artifact, hash_sig_artifact): + service_name = service.name + log_file = lean_shared.lean_log_file_path(service_name) + + # Stage the genesis text files into the running container — Kurtosis + # doesn't allow remounting a new file artifact on an existing service, + # so we read each file via plan.run_sh (yielding a runtime future) and + # plan.exec a heredoc that resolves the future at apply time. + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=["/bin/sh", "-c", "mkdir -p {0}".format(GENESIS_MOUNT)], + ), + description="Preparing genesis mount on {0}".format(service_name), + ) + for filename in GENESIS_TEXT_FILES: + read = plan.run_sh( + run="cat /src/{0}".format(filename), + files={"/src": genesis_artifact}, + description="Reading {0} for {1}".format(filename, service_name), + ) + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=[ + "/bin/sh", + "-c", + "cat > {0}/{1} <<'REAM_EOF'\n{2}\nREAM_EOF".format( + GENESIS_MOUNT, + filename, + read.output, + ), + ], + ), + description="Staging {0} into {1}".format(filename, service_name), + ) + + cmd_parts = [ + ENTRYPOINT, + "--data-dir", + DATA_DIR, + "lean_node", + "--network", + "{0}/config.yaml".format(GENESIS_MOUNT), + "--validator-registry-path", + "{0}/annotated_validators.yaml".format(GENESIS_MOUNT), + "--bootnodes", + "{0}/nodes.yaml".format(GENESIS_MOUNT), + "--node-id", + node["node_name"], + "--node-key", + "{0}/{1}.key".format(NODE_KEY_MOUNT, node["node_name"]), + "--socket-port", + str(constants.LEAN_QUIC_PORT_NUM), + "--metrics", + "--metrics-address", + "0.0.0.0", + "--metrics-port", + str(constants.LEAN_METRICS_PORT_NUM), + "--http-address", + "0.0.0.0", + "--http-port", + str(constants.LEAN_API_PORT_NUM), + ] + if node["is_aggregator"]: + cmd_parts.append("--is-aggregator") + for extra in node["extra_params"]: + cmd_parts.append(extra) + + rust_log = "" + if node["log_level"] != "": + rust_log = "RUST_LOG='{0}' ".format(node["log_level"]) + + nohup_cmd = "setsid -f sh -c \"exec {0}{1}\" < /dev/null >> {2} 2>&1".format( + rust_log, + " ".join(cmd_parts), + log_file, + ) + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=["/bin/sh", "-c", nohup_cmd], + ), + description="Starting ream binary on {0}".format(service_name), + ) + + api_url = "http://{0}:{1}".format(service.ip_address, constants.LEAN_API_PORT_NUM) + metrics_url = "http://{0}:{1}/metrics".format( + service.ip_address, + constants.LEAN_METRICS_PORT_NUM, + ) + + return lean_context.new_lean_context( + client_name=constants.LEAN_TYPE.ream, + service_name=service_name, + ip_address=service.ip_address, + quic_port=constants.LEAN_QUIC_PORT_NUM, + api_port=constants.LEAN_API_PORT_NUM, + metrics_port=constants.LEAN_METRICS_PORT_NUM, + api_url=api_url, + metrics_url=metrics_url, + metrics_info={ + "name": service_name, + "url": metrics_url, + "path": "/metrics", + "config": node["prometheus_config"], + }, + ) diff --git a/src/lean/zeam/zeam_launcher.star b/src/lean/zeam/zeam_launcher.star new file mode 100644 index 000000000..4bc175e71 --- /dev/null +++ b/src/lean/zeam/zeam_launcher.star @@ -0,0 +1,198 @@ +""" +zeam launcher. + +Translates the Lean pipeline's per-node record into zeam's CLI surface from +`client-cmds/zeam-cmd.sh` in blockblaz/lean-quickstart. + +`blockblaz/zeam:devnet4` is a `scratch`-based image — only the zeam binary +exists at `/app/zig-out/bin/zeam`, no `/bin/sh`, `tail`, `nohup`, or +anything else. The Lean pipeline's placeholder-then-plan.exec pattern +needs a shell to (a) keep the placeholder alive while genesis runs, and +(b) cat-stage text genesis files in. To unblock this without rebuilding +zeam's image, we inject a static busybox binary as a file artifact and +mount it at `/bin/busybox`, then drive everything through it. +""" + +constants = import_module("../../package_io/constants.star") +lean_shared = import_module("../lean_shared.star") +lean_context = import_module("../lean_context.star") + +ENTRYPOINT = "/app/zig-out/bin/zeam" +GENESIS_MOUNT = constants.LEAN_GENESIS_MOUNTPOINT_ON_CLIENTS +HASH_SIG_MOUNT = GENESIS_MOUNT + "/hash-sig-keys" +DATA_DIR = "/data" +NODE_KEY_MOUNT = constants.LEAN_NODE_KEY_MOUNTPOINT_ON_CLIENTS +BUSYBOX_MOUNT = "/usr/local/bin" +BUSYBOX = "/usr/local/bin/busybox" + +GENESIS_TEXT_FILES = [ + "config.yaml", + "annotated_validators.yaml", + "nodes.yaml", + "validator-config.yaml", + # zeam scans the custom-genesis dir for validators.yaml (PK's raw + # validator-index assignments); the heredoc-stage covers it since it's + # plain YAML, no binary content. + "validators.yaml", +] + + +def _busybox_artifact(plan): + # Extract the static busybox binary out of busybox:musl. Once exported + # as a Kurtosis files artifact it can be mounted into any scratch + # container as `/usr/local/bin/busybox` so we have a working shell to + # run plan.exec scripts against. + return plan.run_sh( + run="mkdir -p /out && cp /bin/busybox /out/busybox", + image="busybox:musl", + store=[StoreSpec(src="/out", name="lean-busybox")], + description="Extracting static busybox for zeam scratch image", + ).files_artifacts[0] + + +def initialize(plan, node, p2p_keys_artifact, hash_sig_artifact): + busybox_artifact = _busybox_artifact(plan) + cfg_kwargs = lean_shared.common_cfg_kwargs(node) + cfg_kwargs.update( + { + "image": node["image"], + # Override the zeam entrypoint with busybox sh; the real zeam + # binary is invoked later via plan.exec. + "entrypoint": [BUSYBOX, "sh", "-c"], + "cmd": [ + # zeam's scratch image has no /usr/bin/touch, /bin/tail, and + # not even /var/log. Every applet has to be dispatched through + # busybox; mkdir -p creates /var/log on first touch. + "{0} mkdir -p $({0} dirname {1}) && {0} touch {1} && {0} tail -f {1}".format( + BUSYBOX, + lean_shared.lean_log_file_path(node["service_name"]), + ) + ], + "files": { + NODE_KEY_MOUNT: p2p_keys_artifact, + HASH_SIG_MOUNT: hash_sig_artifact, + BUSYBOX_MOUNT: busybox_artifact, + }, + } + ) + return plan.add_service(node["service_name"], ServiceConfig(**cfg_kwargs)) + + +def start(plan, node, service, genesis_artifact, hash_sig_artifact): + service_name = service.name + log_file = lean_shared.lean_log_file_path(service_name) + + # Use busybox sh for every shell-needing step inside the zeam container. + # Pre-create /data so zeam can write its RocksDB / LMDB there, and stage + # the libp2p node key into /network-configs (lean-quickstart's zeam + # contract has --node-key inside the custom-genesis dir, not in a + # separate mount). + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=[ + BUSYBOX, + "sh", + "-c", + "{0} mkdir -p {1} {2} && {0} cp {3}/{4}.key {1}/{4}.key".format( + BUSYBOX, + GENESIS_MOUNT, + DATA_DIR, + NODE_KEY_MOUNT, + node["node_name"], + ), + ], + ), + description="Preparing genesis mount on {0}".format(service_name), + ) + for filename in GENESIS_TEXT_FILES: + read = plan.run_sh( + run="cat /src/{0}".format(filename), + files={"/src": genesis_artifact}, + description="Reading {0} for {1}".format(filename, service_name), + ) + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=[ + BUSYBOX, + "sh", + "-c", + "{3} cat > {0}/{1} <<'ZEAM_EOF'\n{2}\nZEAM_EOF".format( + GENESIS_MOUNT, + filename, + read.output, + BUSYBOX, + ), + ], + ), + description="Staging {0} into {1}".format(filename, service_name), + ) + + cmd_parts = [ + ENTRYPOINT, + "node", + "--custom-genesis", + GENESIS_MOUNT, + # zeam's --validator-config accepts either a directory of per-node + # validator configs OR the literal sentinel `genesis_bootnode`, + # which tells zeam to derive its validator set from the + # GENESIS_VALIDATORS list in config.yaml. Pointing at a single + # file path (lean-quickstart's `validator-config.yaml`) trips + # zeam's "NotDir" check, so use the sentinel here. + "--validator-config", + "genesis_bootnode", + "--data-dir", + DATA_DIR, + "--node-id", + node["node_name"], + # zeam (per lean-quickstart's contract) reads --node-key relative to + # the custom-genesis dir; we staged it there in the prepare step. + "--node-key", + "{0}/{1}.key".format(GENESIS_MOUNT, node["node_name"]), + "--metrics-enable", + "--api-port", + str(constants.LEAN_API_PORT_NUM), + "--metrics-port", + str(constants.LEAN_METRICS_PORT_NUM), + ] + if node["is_aggregator"]: + cmd_parts.append("--is-aggregator") + for extra in node["extra_params"]: + cmd_parts.append(extra) + + nohup_cmd = "{0} nohup {1} >> {2} 2>&1 &".format( + BUSYBOX, + " ".join(cmd_parts), + log_file, + ) + plan.exec( + service_name=service_name, + recipe=ExecRecipe( + command=[BUSYBOX, "sh", "-c", nohup_cmd], + ), + description="Starting zeam binary on {0}".format(service_name), + ) + + api_url = "http://{0}:{1}".format(service.ip_address, constants.LEAN_API_PORT_NUM) + metrics_url = "http://{0}:{1}/metrics".format( + service.ip_address, + constants.LEAN_METRICS_PORT_NUM, + ) + + return lean_context.new_lean_context( + client_name=constants.LEAN_TYPE.zeam, + service_name=service_name, + ip_address=service.ip_address, + quic_port=constants.LEAN_QUIC_PORT_NUM, + api_port=constants.LEAN_API_PORT_NUM, + metrics_port=constants.LEAN_METRICS_PORT_NUM, + api_url=api_url, + metrics_url=metrics_url, + metrics_info={ + "name": service_name, + "url": metrics_url, + "path": "/metrics", + "config": node["prometheus_config"], + }, + ) diff --git a/src/package_io/constants.star b/src/package_io/constants.star index edbdfec61..3bb9f2b0a 100644 --- a/src/package_io/constants.star +++ b/src/package_io/constants.star @@ -20,6 +20,58 @@ CL_TYPE = struct( grandine="grandine", consensoor="consensoor", caplin="caplin", + # Lean Ethereum CL clients. These live in `participants:` alongside the + # standard CL types but are launched by the Lean pipeline under + # `src/lean/` rather than `src/cl/`. The `cl_launcher.star` dispatcher + # skips Lean cl_types; the Lean launcher is then invoked from main.star + # with the participant's el_context (Some when paired with an EL, None + # when el_type is `none`). + # + # ethlambda is the only Lean client with Engine API today + # (lambdaclass/ethlambda#367). The rest run with `el_type: none`. + # `lean_grandine`/`lean_lighthouse` are prefixed to disambiguate from + # the standard Eth1 CLs above which share those repository names. + ethlambda="ethlambda", + ream="ream", + zeam="zeam", + qlean="qlean", + lantern="lantern", + gean="gean", + lean_grandine="lean_grandine", + lean_lighthouse="lean_lighthouse", +) + +# Set of CL types that route to the Lean pipeline. The cl_launcher +# dispatcher skips these (no eth-side beacon launch); main.star then +# builds Lean records from each such participant and hands them to +# lean_launcher.launch with the paired el_context (None for unpaired). +LEAN_CL_TYPES = ( + CL_TYPE.ethlambda, + CL_TYPE.ream, + CL_TYPE.zeam, + CL_TYPE.qlean, + CL_TYPE.lantern, + CL_TYPE.gean, + CL_TYPE.lean_grandine, + CL_TYPE.lean_lighthouse, +) + +# Lean Ethereum consensus clients (post-quantum signatures, leanchain +# genesis, libp2p QUIC). Today only ethlambda implements Engine API on +# the Lean side, so the other clients run with `el_type: none`. The +# `grandine`/`lighthouse` values are prefixed with `lean_` to avoid +# colliding with the Eth1 CL types in CL_TYPE that share those names. +LEAN_TYPE = struct( + ethlambda="ethlambda", + ream="ream", + zeam="zeam", + qlean="qlean", + lantern="lantern", + grandine="lean_grandine", + lighthouse="lean_lighthouse", + gean="gean", + peam="peam", + nlean="nlean", ) VC_TYPE = struct( @@ -110,6 +162,31 @@ DEFAULT_BOOTNODOOR_IMAGE = "ethpandaops/bootnodoor:latest" DEFAULT_ETHEREUM_GENESIS_GENERATOR_IMAGE = ( "ethpandaops/ethereum-genesis-generator:6.0.6" ) + +# Lean genesis tooling. `pk910-leanchain` is pk910's leanchain branch of +# eth-beacon-genesis (ethpandaops/eth-beacon-genesis PR #36); it consumes a +# validator-config.yaml and emits config.yaml + validators.yaml + nodes.yaml + +# genesis.{ssz,json}. `hash-sig-cli` generates the XMSS attester/proposer +# keypairs that GENESIS_VALIDATORS references. +DEFAULT_LEAN_GENESIS_GENERATOR_IMAGE = "ethpandaops/eth-beacon-genesis:pk910-leanchain" +DEFAULT_LEAN_HASH_SIG_CLI_IMAGE = "blockblaz/hash-sig-cli:latest" + +# Lean P2P / API / metrics port IDs and defaults. Lean clients speak QUIC over +# UDP only (no TCP discovery), expose a JSON REST API, and a Prometheus metrics +# endpoint on a separate port — the same triple used by every Lean client in +# blockblaz/lean-quickstart. +LEAN_QUIC_PORT_ID = "quic" +LEAN_API_PORT_ID = "http" +LEAN_METRICS_PORT_ID = "metrics" +LEAN_QUIC_PORT_NUM = 9000 +LEAN_API_PORT_NUM = 5052 +LEAN_METRICS_PORT_NUM = 5054 + +# Mountpoints inside Lean client containers. Kept stable across all Lean +# clients so the integration contract documented in docs/lean-adding-a-new-client.md +# matches what clients receive at runtime. +LEAN_GENESIS_MOUNTPOINT_ON_CLIENTS = "/network-configs" +LEAN_NODE_KEY_MOUNTPOINT_ON_CLIENTS = "/node-keys" DEFAULT_YQ_IMAGE = "linuxserver/yq" DEFAULT_FLASHBOTS_RELAY_IMAGE = "ethpandaops/mev-boost-relay:main" DEFAULT_FLASHBOTS_BUILDER_IMAGE = "ethpandaops/reth-rbuilder:develop" diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index 7ab3732a7..03b1f7c0d 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -26,6 +26,17 @@ DEFAULT_CL_IMAGES = { "grandine": "sifrai/grandine:stable", "consensoor": "ethpandaops/consensoor:main", "caplin": "ethpandaops/caplin:main", + # Lean CL clients. Routed through the Lean pipeline (src/lean/), not + # the standard CL launchers. Operators pin a specific devnet image + # per-participant via `cl_image:` in their args file. + "ethlambda": "ghcr.io/lambdaclass/ethlambda:latest", + "ream": "ghcr.io/reamlabs/ream:latest", + "zeam": "blockblaz/zeam:latest", + "qlean": "qdrvm/qlean-mini:latest", + "lantern": "piertwo/lantern:latest", + "gean": "ghcr.io/geanlabs/gean:latest", + "lean_grandine": "sifrai/lean:latest", + "lean_lighthouse": "hopinheimer/lighthouse:latest", } DEFAULT_CL_IMAGES_MINIMAL = { @@ -37,6 +48,14 @@ DEFAULT_CL_IMAGES_MINIMAL = { "grandine": "ethpandaops/grandine:develop-minimal", "consensoor": "ethpandaops/consensoor:main", "caplin": "ethpandaops/caplin:main", + "ethlambda": "ghcr.io/lambdaclass/ethlambda:latest", + "ream": "ghcr.io/reamlabs/ream:latest", + "zeam": "blockblaz/zeam:latest", + "qlean": "qdrvm/qlean-mini:latest", + "lantern": "piertwo/lantern:latest", + "gean": "ghcr.io/geanlabs/gean:latest", + "lean_grandine": "sifrai/lean:latest", + "lean_lighthouse": "hopinheimer/lighthouse:latest", } DEFAULT_VC_IMAGES = { @@ -73,6 +92,7 @@ DEFAULT_ADDITIONAL_SERVICES = [] ATTR_TO_BE_SKIPPED_AT_ROOT = ( "network_params", "participants", + "lean_network_params", "mev_params", "blockscout_params", "dora_params", @@ -137,6 +157,15 @@ def input_parser(plan, input_args): result["zkboost_params"] = get_default_zkboost_params() result["buildoor_params"] = get_default_buildoor_params() + # Lean Ethereum: knobs (active_epoch, attestation_committee_count, + # num_validator_keys_per_node, metrics_enabled, ...) live in their + # own params block since they don't map cleanly to Eth1 fields. Only + # consulted when at least one participant has a Lean cl_type. + result["lean_network_params"] = default_lean_network_params() + if "lean_network_params" in input_args: + for k, v in input_args["lean_network_params"].items(): + result["lean_network_params"][k] = v + if constants.NETWORK_NAME.shadowfork in result["network_params"]["network"]: shadow_base = result["network_params"]["network"].split("-shadowfork")[0] result["network_params"][ @@ -442,7 +471,20 @@ def input_parser(plan, input_args): ) ) - if result["network_params"]["fulu_fork_epoch"] != constants.FAR_FUTURE_EPOCH: + # Fulu / PeerDAS validation only applies to the Eth1 EL/CL participant + # network. Skip it when every participant has a Lean cl_type — Lean + # clients don't speak PeerDAS. + has_any_eth1_cl = any( + [ + p["cl_type"] not in constants.LEAN_CL_TYPES + for p in result["participants"] + ] + ) + if ( + result["network_params"]["fulu_fork_epoch"] != constants.FAR_FUTURE_EPOCH + and len(result["participants"]) > 0 + and has_any_eth1_cl + ): has_supernodes = False has_node_with_128_plus_validators = False num_perfect_peerdas_participants = 0 @@ -595,8 +637,18 @@ def input_parser(plan, input_args): _validate_ere_gpu_config(result["zkboost_params"]["zkvms"]) + # The "first participant must have an EL" check only applies when there + # actually IS at least one Eth1 participant; lean-only deployments + # (participants: []) skip the EL/CL pipeline entirely. We also skip it + # when every participant has a Lean cl_type — Lean clients use their + # own libp2p QUIC mesh and don't need an Eth1 bootnode. + all_lean = len(result["participants"]) > 0 and all( + [p["cl_type"] in constants.LEAN_CL_TYPES for p in result["participants"]] + ) if ( - "bootnodoor" not in result["additional_services"] + len(result["participants"]) > 0 + and not all_lean + and "bootnodoor" not in result["additional_services"] and result["participants"][0]["el_type"] == constants.EL_TYPE.none ): fail( @@ -715,6 +767,7 @@ def input_parser(plan, input_args): vc_beacon_node_indices=participant["vc_beacon_node_indices"], checkpoint_sync_enabled=participant["checkpoint_sync_enabled"], skip_start=participant["skip_start"], + is_aggregator=participant["is_aggregator"], ) for participant in result["participants"] ], @@ -1099,6 +1152,10 @@ def input_parser(plan, input_args): builder_api=result["buildoor_params"]["builder_api"], epbs_builder=result["buildoor_params"]["epbs_builder"], ), + # Lean Ethereum knobs. Stored as a plain dict (not a nested struct) + # because the Lean per-client launchers reach for fields by string + # key — see src/lean/lean_launcher.star. + lean_network_params=result["lean_network_params"], ) @@ -1829,6 +1886,9 @@ def default_participant(): "vc_beacon_node_indices": None, "checkpoint_sync_enabled": None, "skip_start": False, + # Lean CL knob — only consulted when cl_type is in constants.LEAN_CL_TYPES. + # Non-Lean participants ignore it. + "is_aggregator": False, } @@ -2552,3 +2612,51 @@ def get_devnet_modified_images(network_name, default_images): modified_images[client_type] = get_devnet_image_tag(network_name, image) return modified_images + + +# --------------------------------------------------------------------------- +# Lean Ethereum parsing +# --------------------------------------------------------------------------- +# Lean is a post-quantum-signature consensus stack with its own genesis +# pipeline (PK's eth-beacon-genesis leanchain). Lean clients live in the +# standard `participants:` block with a Lean `cl_type` value (see +# constants.LEAN_CL_TYPES). main.star synthesizes Lean records from each +# such participant and hands them to src/lean/lean_launcher.star, which +# runs the XMSS / hash-sig / leanchain genesis pipeline and starts the +# client binary. Network-level knobs (active_epoch, +# attestation_committee_count, ...) live in `lean_network_params:`. + + +def default_lean_network_params(): + # Genesis timing and shape parameters consumed by the Lean genesis tool + # (eth-beacon-genesis leanchain). Keep them flat to mirror the + # `validator-config.yaml.config` block expected by the generator and by + # every Lean client's CLI. + return { + # Seconds added to "now" to compute GENESIS_TIME when the user does + # not pass an absolute genesis_time. 60s gives all containers time to + # boot, mount artifacts, and reach the gossip mesh before slot 0. + "genesis_delay": 60, + # Explicit Unix timestamp; 0 = derive from genesis_delay. + "genesis_time": 0, + # leanSpec ATTESTATION_COMMITTEE_COUNT. 1 = single committee per slot + # (the only configuration covered by the spec tests today). + "attestation_committee_count": 1, + # log_2(active epochs) for the XMSS hash-sig scheme. 18 matches the + # default in lean-quickstart's validator-config.yaml. + "active_epoch": 18, + # Number of validator hash-sig keypairs to assign to each node when + # the participant does not override `validator_count`. + "num_validator_keys_per_node": 1, + # Image overrides for the Lean genesis tooling. Empty = use the + # `DEFAULT_LEAN_GENESIS_GENERATOR_IMAGE` / `DEFAULT_LEAN_HASH_SIG_CLI_IMAGE` + # constants. Override to pin a specific PK genesis-tool commit. + "genesis_generator_image": "", + "hash_sig_cli_image": "", + # When true, start a Prometheus + Grafana stack inside the enclave + # that scrapes every Lean node's `/metrics` endpoint and serves the + # upstream Lean client dashboard at port 3000. + "metrics_enabled": True, + } + + diff --git a/src/package_io/sanity_check.star b/src/package_io/sanity_check.star index 8af4d1ab0..7d1ff6d9e 100644 --- a/src/package_io/sanity_check.star +++ b/src/package_io/sanity_check.star @@ -77,6 +77,7 @@ PARTICIPANT_CATEGORIES = { "vc_beacon_node_indices", "checkpoint_sync_enabled", "skip_start", + "is_aggregator", ], } @@ -487,6 +488,11 @@ ADDITIONAL_SERVICES_PARAMS = [ ] ADDITIONAL_CATEGORY_PARAMS = { + # Lean Ethereum network-level knobs (active_epoch, + # attestation_committee_count, ...). Per-participant Lean config + # rides on `participants:` entries with a Lean cl_type — see + # constants.LEAN_CL_TYPES. + "lean_network_params": "", "wait_for_finalization": "", "global_log_level": "", "snooper_enabled": "", diff --git a/src/participant_network.star b/src/participant_network.star index 1b4c0dd8e..7a422f9eb 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -348,6 +348,21 @@ def launch_participant_network( index_str = shared_utils.zfill_custom( index + 1, len(str(len(args_with_right_defaults.participants))) ) + # Lean cl_types own their own validator client logic (XMSS keys live + # inside the consensus binary). Skip the entire VC / remote-signer / + # snooper / metrics-exporter pipeline for them — those things assume + # an Eth1 beacon API that Lean clients don't expose. Pad the context + # lists that get appended-to in-loop so per-index alignment with + # `participants` holds. (`all_vc_contexts` is rebuilt below, so it + # doesn't need padding here.) + if cl_type in constants.LEAN_CL_TYPES: + all_remote_signer_contexts.append(None) + all_snooper_beacon_contexts.append(None) + all_snooper_el_rpc_contexts.append(None) + all_ethereum_metrics_exporter_contexts.append(None) + all_xatu_sentry_contexts.append(None) + continue + el_context = all_el_contexts[index] if index < len(all_el_contexts) else None cl_context = all_cl_contexts[index] if index < len(all_cl_contexts) else None diff --git a/src/prelaunch_data_generator/lean_genesis/lean_genesis_generator.star b/src/prelaunch_data_generator/lean_genesis/lean_genesis_generator.star new file mode 100644 index 000000000..9ff879ba4 --- /dev/null +++ b/src/prelaunch_data_generator/lean_genesis/lean_genesis_generator.star @@ -0,0 +1,531 @@ +""" +Lean Ethereum genesis generation. + +This module orchestrates the post-quantum genesis pipeline used by every +Lean client: + + 1. Generate XMSS attester+proposer keypairs via `blockblaz/hash-sig-cli`. + 2. Render `validator-config.yaml` from the live (Kurtosis-assigned) IPs and + ports of each Lean participant. + 3. Run `ethpandaops/eth-beacon-genesis:pk910-leanchain` to derive + `config.yaml`, `validators.yaml`, `nodes.yaml`, and `genesis.{ssz,json}`. + 4. Post-process: inject GENESIS_VALIDATORS into config.yaml and render + `annotated_validators.yaml` (node-name -> validator-index assignments + with attester/proposer privkey filenames). + +The output is a single files artifact (`lean-genesis-data`) mounted at +`/network-configs` inside every Lean client container, plus a separate +`lean-hash-sig-keys` artifact for the XMSS secret/public keys. This matches +the `lean-quickstart` on-disk layout 1:1 so a client written for +lean-quickstart works under Kurtosis without code changes. +""" + +constants = import_module("../../package_io/constants.star") + +GENESIS_ARTIFACT_NAME = "lean-genesis-data" +HASH_SIG_ARTIFACT_NAME = "lean-hash-sig-keys" + +GENESIS_DIR = "/genesis" +HASH_SIG_DIR = "/hash-sig-keys" + + +def _resolve_images(lean_network_params): + genesis_image = lean_network_params.get("genesis_generator_image", "") + if genesis_image == "": + genesis_image = constants.DEFAULT_LEAN_GENESIS_GENERATOR_IMAGE + hash_sig_image = lean_network_params.get("hash_sig_cli_image", "") + if hash_sig_image == "": + hash_sig_image = constants.DEFAULT_LEAN_HASH_SIG_CLI_IMAGE + return genesis_image, hash_sig_image + + +def _compute_genesis_time(plan, lean_network_params): + """Resolve GENESIS_TIME. + + Explicit `genesis_time` wins; otherwise we ask a busybox shell for + `now() + genesis_delay`. Runs on the Kurtosis backend so the result is + a real clock reading inside the cluster, not the operator's laptop. + """ + explicit = lean_network_params.get("genesis_time", 0) + if explicit != 0: + return str(explicit) + delay = lean_network_params.get("genesis_delay", 60) + result = plan.run_sh( + run="echo -n $(($(date +%s) + {0}))".format(delay), + description="Computing Lean genesis time", + ) + return result.output + + +def _render_validator_config(plan, services_meta, lean_network_params, keys_artifact): + """Render validator-config.yaml from per-node IPs / ports / keys. + + Two-stage render: + + Stage 1 - `plan.render_templates` produces validator-config.yaml with + IPs and ports substituted from Kurtosis runtime futures, but the + privkey field set to a placeholder marker `__PRIVKEY___`. + Kurtosis's template engine handles the IP futures correctly; trying + to embed them inside a raw shell heredoc doesn't (the {{kurtosis:...}} + markers reach the shell before substitution and break sh parsing). + + Stage 2 - `plan.run_sh` reads the placeholders and replaces each one + with the corresponding key from the lean-node-keys artifact via sed. + The shell never sees a Kurtosis future, only literal text. + + `services_meta` is a list of dicts with keys: name, ip_address, + quic_port, metrics_port, api_port, validator_count, is_aggregator. + """ + template = """shuffle: roundrobin +deployment_mode: kurtosis +config: + activeEpoch: {{.ActiveEpoch}} + keyType: "hash-sig" + attestation_committee_count: {{.AttestationCommitteeCount}} +validators: +{{- range .Validators}} + - name: "{{.Name}}" + privkey: "__PRIVKEY_{{.Name}}__" + enrFields: + ip: "{{.Ip}}" + quic: {{.Quic}} + metricsPort: {{.MetricsPort}} + apiPort: {{.ApiPort}} + isAggregator: {{.IsAggregator}} + count: {{.Count}} +{{end}}""" + + validators = [] + for meta in services_meta: + validators.append( + { + "Name": meta["name"], + "Ip": meta["ip_address"], + "Quic": meta["quic_port"], + "MetricsPort": meta["metrics_port"], + "ApiPort": meta["api_port"], + "IsAggregator": "true" if meta["is_aggregator"] else "false", + "Count": meta["validator_count"], + } + ) + + template_artifact = plan.render_templates( + config={ + "validator-config.yaml": struct( + template=template, + data={ + "ActiveEpoch": lean_network_params["active_epoch"], + "AttestationCommitteeCount": lean_network_params[ + "attestation_committee_count" + ], + "Validators": validators, + }, + ), + }, + name="lean-validator-config-template", + description="Rendering Lean validator-config.yaml (stage 1, placeholders)", + ) + + # Stage 2: sed each `__PRIVKEY___` to the contents of the + # matching `/keys/.key`. Run inside a single shell so we don't + # have to thread Kurtosis futures through more steps. + sed_lines = ["set -eu", "mkdir -p /out", "cp /tpl/validator-config.yaml /out/"] + for meta in services_meta: + # The privkey file holds a raw 64-char hex string with no surrounding + # whitespace. Use sed with a `|` delimiter since the key is hex + # (which never contains `|`) and the placeholder is unique. + sed_lines.append( + ( + "key=$(cat /keys/{0}.key) && " + + 'sed -i "s|__PRIVKEY_{0}__|$key|" /out/validator-config.yaml' + ).format(meta["name"]) + ) + sed_script = "\n".join(sed_lines) + + result = plan.run_sh( + run=sed_script, + # alpine/openssl already lives in the engine cache from the P2P + # key generation step and ships busybox sed. + image="alpine/openssl", + files={ + "/tpl": template_artifact, + "/keys": keys_artifact, + }, + store=[ + StoreSpec(src="/out/validator-config.yaml", name="lean-validator-config") + ], + description="Rendering Lean validator-config.yaml (stage 2, privkey inlining)", + ) + return result.files_artifacts[0] + + +def _render_initial_config(plan, genesis_time, lean_network_params, total_validators): + """Render the initial config.yaml that PK's tool consumes. + + PK's tool *rewrites* config.yaml with extra fields after running, but it + still requires the input to declare GENESIS_TIME, ATTESTATION_COMMITTEE_COUNT, + ACTIVE_EPOCH, and VALIDATOR_COUNT — every other Lean client reads + GENESIS_TIME from this file too, so the value must match what we'll embed. + """ + template = """# Genesis Settings +GENESIS_TIME: {{.GenesisTime}} + +# Chain Settings +ATTESTATION_COMMITTEE_COUNT: {{.AttestationCommitteeCount}} + +# Key Settings +ACTIVE_EPOCH: {{.ActiveEpoch}} + +# Validator Settings +VALIDATOR_COUNT: {{.ValidatorCount}} +""" + return plan.render_templates( + config={ + "config.yaml": struct( + template=template, + data={ + "GenesisTime": genesis_time, + "AttestationCommitteeCount": lean_network_params[ + "attestation_committee_count" + ], + "ActiveEpoch": lean_network_params["active_epoch"], + "ValidatorCount": total_validators, + }, + ), + }, + name="lean-initial-config", + description="Rendering Lean initial config.yaml", + ) + + +def _generate_hash_sig_keys(plan, image, num_validators, active_epoch): + """Generate XMSS attester+proposer keypairs. + + `hash-sig-cli generate` writes `validator_N_{attester,proposer}_key_{pk,sk}.ssz` + plus a `validator-keys-manifest.yaml` index. The whole `/hash-sig-keys` + tree becomes the `lean-hash-sig-keys` artifact mounted at the Lean + client's `--hash-sig-keys-dir`. + """ + # The binary in `blockblaz/hash-sig-cli:latest` is `hashsig` (the image's + # ENTRYPOINT). Kurtosis's `plan.run_sh` overrides the entrypoint with + # `sh -c`, so we have to call the binary by its absolute path. It lives + # at `/usr/local/bin/hashsig`. + plan.run_sh( + run=( + "mkdir -p {0} && " + + "/usr/local/bin/hashsig generate " + + "--num-validators {1} " + + "--log-num-active-epochs {2} " + + "--output-dir {0} " + + "--export-format ssz" + ).format(HASH_SIG_DIR, num_validators, active_epoch), + image=image, + # XMSS keygen is CPU-bound and scales with --num-validators * + # 2^active_epoch. On slower hosts (or with active_epoch >= 18) it + # overruns the default 180s plan.run_sh timeout, so give it 30 + # minutes. The step is idempotent — re-runs won't regenerate. + wait="30m", + store=[ + StoreSpec(src=HASH_SIG_DIR, name=HASH_SIG_ARTIFACT_NAME), + ], + description="Generating Lean hash-sig validator keys ({0} validators)".format( + num_validators + ), + ) + return HASH_SIG_ARTIFACT_NAME + + +def generate_hash_sig_keys(plan, lean_network_params, total_validators): + """Public wrapper around _generate_hash_sig_keys. + + Exposed so `lean_launcher.launch` can pre-create the hash-sig artifact + before any service is added. This lets every Lean client mount the keys + at `initialize()` time (the keys are IP-independent), keeping us inside + Kurtosis's "one add_service per name" constraint. + """ + if total_validators < 1: + fail( + "Lean genesis requires at least one validator across all " + + "Lean participants (got 0).", + ) + _, hash_sig_image = _resolve_images(lean_network_params) + return _generate_hash_sig_keys( + plan, + hash_sig_image, + total_validators, + lean_network_params["active_epoch"], + ) + + +def _run_genesis_tool( + plan, + image, + validator_config_artifact, + initial_config_artifact, +): + """Invoke pk910-leanchain. Outputs config.yaml/validators.yaml/nodes.yaml/genesis.{ssz,json}. + + The tool reads `validator-config.yaml` to figure out per-node ENRs and + validator counts, and writes the canonical genesis bundle. We mount the + pre-rendered config.yaml from `--config-output` and let the tool overwrite + it in place; this lets the tool inject the fork digests and other fields + it computes itself. + """ + plan.run_sh( + run=( + "mkdir -p {0} && " + + "cp /input-validator/validator-config.yaml {0}/validator-config.yaml && " + + "cp /input-config/config.yaml {0}/config.yaml && " + + "/app/eth-genesis-state-generator leanchain " + + "--config {0}/config.yaml " + + "--mass-validators {0}/validator-config.yaml " + + "--state-output {0}/genesis.ssz " + + "--json-output {0}/genesis.json " + + "--nodes-output {0}/nodes.yaml " + + "--validators-output {0}/validators.yaml " + + "--config-output {0}/config.yaml" + ).format(GENESIS_DIR), + image=image, + files={ + "/input-validator": validator_config_artifact, + "/input-config": initial_config_artifact, + }, + # The genesis tool's outputs are merged with the hash-sig keys and + # post-processing additions further down before the final artifact + # is published. + store=[ + StoreSpec(src=GENESIS_DIR, name="lean-genesis-raw"), + ], + description="Running eth-beacon-genesis leanchain", + ) + return "lean-genesis-raw" + + +def _post_process( + plan, + raw_genesis_artifact, + hash_sig_artifact, + validator_config_artifact, + node_key_artifact, +): + """Bundle everything Lean clients need into a single mountable artifact. + + Steps: + * Append GENESIS_VALIDATORS to config.yaml using the dual-key + manifest emitted by hash-sig-cli (attester_key_pubkey_hex + + proposer_key_pubkey_hex per validator). + * Render annotated_validators.yaml mapping node names to validator + indices and their `_attester_` / `_proposer_` privkey file basenames + (ethlambda, lantern, and grandine all parse this exact filename + convention to route keys to attestation vs proposal slots). + * Copy the per-node P2P keys (`.key`) into the same artifact so + every client just mounts `/network-configs` and reads everything it + needs from one place. + + Implemented as a Python script (Starlark has no YAML libs and busybox + `sh` in common images chokes on heredocs with embedded interpreters). + The script is rendered as a separate artifact via `render_templates` + and invoked by a tiny shell wrapper - no heredocs reach `sh`. + """ + # The script reads the hash-sig manifest, PK's validators.yaml output, + # and the raw genesis bundle, then writes: + # - /out/config.yaml with GENESIS_VALIDATORS appended (dual-key layout) + # - /out/annotated_validators.yaml (node_name -> [{index, pubkey_hex, + # privkey_file}, ...] with attester + proposer rows per index) + # - all other genesis files copied through unchanged + # - hash-sig keys bundled into ./hash-sig-keys/ + # - per-node `.key` libp2p secrets bundled at the top level + python_source = """import os +import shutil +import yaml + +RAW = "/raw" +HASH_SIG = "/hash-sig" +VC = "/vc" +NODE_KEYS = "/node-keys" +OUT = "/out" +MANIFEST = os.path.join(HASH_SIG, "validator-keys-manifest.yaml") + + +def _as_hex(value): + # Normalise a pubkey field to a no-0x-prefix lowercase hex string. + # Even with BaseLoader (which keeps everything as str) we strip the 0x + # prefix here; int fallback handles unexpected manifest shapes. + if isinstance(value, int): + return format(value, "x") + s = str(value) + if s.startswith("0x") or s.startswith("0X"): + s = s[2:] + return s + + +def copytree_into(src, dst): + os.makedirs(dst, exist_ok=True) + for entry in os.listdir(src): + s = os.path.join(src, entry) + d = os.path.join(dst, entry) + if os.path.isdir(s): + shutil.copytree(s, d, dirs_exist_ok=True) + else: + shutil.copy2(s, d) + + +# Stage 1: bundle all input artifacts into /out. +os.makedirs(OUT, exist_ok=True) +copytree_into(RAW, OUT) +shutil.copy2(os.path.join(VC, "validator-config.yaml"), OUT) +for f in os.listdir(NODE_KEYS): + if f.endswith(".key"): + shutil.copy2(os.path.join(NODE_KEYS, f), OUT) +copytree_into(HASH_SIG, os.path.join(OUT, "hash-sig-keys")) + +# Stage 2: append GENESIS_VALIDATORS (dual-key) to config.yaml. +with open(MANIFEST) as f: + # BaseLoader keeps every scalar as a Python str. We need this for the + # XMSS pubkey hex fields: YAML 1.1 (PyYAML's default) interprets + # unquoted `0x...` tokens as integers, which silently drops leading + # zeros when we format the value back out — clients then reject the + # config because the pubkey has an odd number of hex digits. + manifest = yaml.load(f, Loader=yaml.BaseLoader) + +gv_lines = ["", "# Genesis validator public keys (post-quantum hash-sig)", "GENESIS_VALIDATORS:"] +for v in manifest["validators"]: + ah = _as_hex(v["attester_key_pubkey_hex"]) + ph = _as_hex(v["proposer_key_pubkey_hex"]) + gv_lines.append(' - attestation_pubkey: "{0}"'.format(ah)) + gv_lines.append(' proposal_pubkey: "{0}"'.format(ph)) +with open(os.path.join(OUT, "config.yaml"), "a") as f: + f.write("\\n".join(gv_lines) + "\\n") + +# Stage 3: render annotated_validators.yaml from validators.yaml + manifest. +with open(os.path.join(OUT, "validators.yaml")) as f: + # Use BaseLoader too (see manifest load above) for consistency, even + # though this file has no hex tokens to worry about today. + assignments = yaml.load(f, Loader=yaml.BaseLoader) or {} + +ann_lines = [] +for node, indices in assignments.items(): + ann_lines.append("{0}:".format(node)) + if not indices: + ann_lines.append(" []") + continue + for idx in indices: + v = manifest["validators"][int(idx)] + ah = _as_hex(v["attester_key_pubkey_hex"]) + ph = _as_hex(v["proposer_key_pubkey_hex"]) + ann_lines.append(" - index: {0}".format(idx)) + ann_lines.append(" pubkey_hex: {0}".format(ah)) + ann_lines.append( + " privkey_file: validator_{0}_attester_key_sk.ssz".format(idx) + ) + ann_lines.append(" - index: {0}".format(idx)) + ann_lines.append(" pubkey_hex: {0}".format(ph)) + ann_lines.append( + " privkey_file: validator_{0}_proposer_key_sk.ssz".format(idx) + ) +with open(os.path.join(OUT, "annotated_validators.yaml"), "w") as f: + f.write("\\n".join(ann_lines) + "\\n") +""" + + script_artifact = plan.render_templates( + config={ + "post_process.py": struct(template=python_source, data={}), + }, + name="lean-post-process-script", + description="Rendering Lean genesis post-process script", + ) + + return plan.run_sh( + run=( + "set -eu; " + + "pip install --quiet --root-user-action=ignore pyyaml; " + + "python3 /script/post_process.py" + ), + image="python:3-alpine", + files={ + "/raw": raw_genesis_artifact, + "/hash-sig": hash_sig_artifact, + "/vc": validator_config_artifact, + "/node-keys": node_key_artifact, + "/script": script_artifact, + }, + store=[ + StoreSpec(src="/out", name=GENESIS_ARTIFACT_NAME), + ], + description="Post-processing Lean genesis (GENESIS_VALIDATORS + annotated_validators.yaml)", + ).files_artifacts[0] + + +def generate( + plan, + services_meta, + lean_network_params, + node_key_artifact, + hash_sig_artifact, +): + """Top-level entrypoint. + + Args: + plan: Kurtosis plan. + services_meta: list of dicts (one per node) with: name, ip_address, + quic_port, metrics_port, api_port, validator_count, is_aggregator. + The caller (lean_launcher) builds this list after Kurtosis has + assigned IPs to the placeholder services. + lean_network_params: validated `lean_network_params` block. + node_key_artifact: files artifact holding `.key` ASCII-hex P2P + secrets (one per node). + hash_sig_artifact: files artifact holding the XMSS attester+proposer + keypairs (pre-generated by `generate_hash_sig_keys`). + + Returns: + struct(genesis_artifact = , hash_sig_artifact = , + genesis_time = ). + """ + total_validators = 0 + for meta in services_meta: + total_validators += meta["validator_count"] + + if total_validators < 1: + fail( + "Lean genesis requires at least one validator across all " + + "Lean participants (got 0).", + ) + + genesis_image, _ = _resolve_images(lean_network_params) + genesis_time = _compute_genesis_time(plan, lean_network_params) + + validator_config_artifact = _render_validator_config( + plan, + services_meta, + lean_network_params, + node_key_artifact, + ) + + initial_config_artifact = _render_initial_config( + plan, + genesis_time, + lean_network_params, + total_validators, + ) + + raw_genesis_artifact = _run_genesis_tool( + plan, + genesis_image, + validator_config_artifact, + initial_config_artifact, + ) + + final_artifact = _post_process( + plan, + raw_genesis_artifact, + hash_sig_artifact, + validator_config_artifact, + node_key_artifact, + ) + + return struct( + genesis_artifact=final_artifact, + hash_sig_artifact=hash_sig_artifact, + genesis_time=genesis_time, + total_validators=total_validators, + ) diff --git a/src/prelaunch_data_generator/lean_genesis/p2p_keys_generator.star b/src/prelaunch_data_generator/lean_genesis/p2p_keys_generator.star new file mode 100644 index 000000000..16818fad2 --- /dev/null +++ b/src/prelaunch_data_generator/lean_genesis/p2p_keys_generator.star @@ -0,0 +1,48 @@ +""" +P2P key generation for Lean consensus nodes. + +Each Lean node needs a 32-byte libp2p identity key (the secp256k1 secret that +derives the peer ID and ENR). lean-quickstart writes one such key per node +into `.key` as ASCII hex. We reproduce that exact layout so the +genesis tool and every Lean client's CLI receive the same files at runtime. + +We deliberately do NOT read the generated key values back into Starlark — in +Kurtosis, `plan.run_sh(...).output` is a runtime future that only materialises +when the plan is applied, so it can't be used to drive Starlark interpretation +(dict lookups, loops, etc.). Instead we export the keys as a files artifact +and let *downstream containers* read each key from `/keys/.key` when +they need it. +""" + +OPENSSL_IMAGE = "alpine/openssl" + +KEYS_ARTIFACT_NAME = "lean-node-keys" +KEYS_MOUNT_INSIDE_GENERATOR = "/keys" + + +def generate_node_keys(plan, node_names): + """Generate one 32-byte hex P2P key per node into a single artifact. + + Returns the artifact name. Reading individual key values back into + Starlark is not supported — see the module docstring. + """ + # `tr -d` strips OpenSSL's trailing newline; without it, the genesis + # tool's libp2p peer ID derivation downstream sees an extra byte and + # computes the wrong identity, which then mismatches what the running + # client derives. + script_parts = ["set -eu", "mkdir -p {0}".format(KEYS_MOUNT_INSIDE_GENERATOR)] + for name in node_names: + script_parts.append( + "openssl rand -hex 32 | tr -d '\\n' > {0}/{1}.key".format( + KEYS_MOUNT_INSIDE_GENERATOR, name + ) + ) + script = "\n".join(script_parts) + + plan.run_sh( + run=script, + image=OPENSSL_IMAGE, + store=[StoreSpec(src=KEYS_MOUNT_INSIDE_GENERATOR, name=KEYS_ARTIFACT_NAME)], + description="Generating Lean P2P node keys ({0} nodes)".format(len(node_names)), + ) + return KEYS_ARTIFACT_NAME diff --git a/src/shared_utils/shared_utils.star b/src/shared_utils/shared_utils.star index 10922a4cc..5485aa97b 100644 --- a/src/shared_utils/shared_utils.star +++ b/src/shared_utils/shared_utils.star @@ -209,13 +209,16 @@ def get_client_names(participant, index, participant_contexts, participant_confi cl_client = participant.cl_context el_client = participant.el_context vc_client = participant.vc_context + # Lean cl_types don't go through the standard CL launcher, so cl_context + # is None. Fall back to the cl_type string from the participant config so + # downstream consumers (validator-ranges, dora, etc.) still get a usable + # name for the row. + cl_name = cl_client.client_name if cl_client != None else participant_config.cl_type if el_client == None: - base_name = "{0}-{1}".format(index_str, cl_client.client_name) + base_name = "{0}-{1}".format(index_str, cl_name) else: - base_name = "{0}-{1}-{2}".format( - index_str, el_client.client_name, cl_client.client_name - ) - if vc_client != None and cl_client.client_name != vc_client.client_name: + base_name = "{0}-{1}-{2}".format(index_str, el_client.client_name, cl_name) + if vc_client != None and cl_name != vc_client.client_name: full_name = base_name + "-{0}".format(vc_client.client_name) else: full_name = base_name