Skip to content

revme: add --run regex filter to statetest and blockchaintest #3544

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
spencer-tb wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

revme: add --run regex filter to statetest and blockchaintest #3544

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a37da9c
revme: add --run regex filter to statetest and blockchaintest
spencer-tb Apr 5, 2026
9827b5e
revme: add --json-array flag for standard JSON array output
spencer-tb Apr 5, 2026
db42dc2
revme: add --version flag
spencer-tb Apr 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions bins/revme/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ thiserror.workspace = true
walkdir.workspace = true
k256 = { workspace = true, features = ["ecdsa"] }
csv = "1.1.6"
regex = "1"

[features]
default = ["map-foldhash"]
Expand Down
121 changes: 96 additions & 25 deletions bins/revme/src/cmd/blockchaintest.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ pub mod pre_block;

use crate::dir_utils::find_all_json_tests;
use clap::Parser;
use regex::Regex;

use revm::statetest_types::blockchain::{
Account, BlockchainTest, BlockchainTestCase, ForkSpec, Withdrawal,
Expand Down Expand Up @@ -55,11 +56,26 @@ pub struct Cmd {
/// Output results in JSON format
#[arg(long)]
json: bool,
/// Output results as a JSON array with standard schema to stdout
///
/// Fields: name, pass, fork, stateRoot, error
#[arg(long)]
json_array: bool,
Copy link
Copy Markdown
Member

@rakita rakita Apr 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i am not exactly sure what is the purpose of json_array. We have --json that outputs it in json format that is used for comparison with outher evms

/// Only run tests whose name matches this regex
#[arg(long)]
run: Option<String>,
}

impl Cmd {
/// Runs `blockchaintest` command.
pub fn run(&self) -> Result<(), Error> {
let run_filter = self
.run
.as_deref()
.map(Regex::new)
.transpose()
.map_err(|e| Error::RegexError(e.to_string()))?;

for path in &self.paths {
if !path.exists() {
return Err(Error::PathNotFound(path.clone()));
Expand All @@ -80,6 +96,8 @@ impl Cmd {
self.keep_going,
self.print_env_on_error,
self.json,
self.json_array,
run_filter.as_ref(),
)?;
}
Ok(())
Expand All @@ -93,11 +111,15 @@ fn run_tests(
keep_going: bool,
print_env_on_error: bool,
json_output: bool,
json_array: bool,
run_filter: Option<&Regex>,
) -> Result<(), Error> {
let mut passed = 0;
let mut failed = 0;
let mut skipped = 0;
let mut failed_paths = Vec::new();
let mut json_array_results: Vec<serde_json::Value> = Vec::new();
let keep_going = keep_going || json_array;

let start_time = Instant::now();
let total_files = test_files.len();
Expand All @@ -124,7 +146,14 @@ fn run_tests(
continue;
}

let result = run_test_file(&file_path, json_output, print_env_on_error);
let result = run_test_file(
&file_path,
json_output,
json_array,
print_env_on_error,
run_filter,
&mut json_array_results,
);

match result {
Ok(test_count) => {
Expand Down Expand Up @@ -172,6 +201,14 @@ fn run_tests(

let duration = start_time.elapsed();

if json_array {
println!(
"{}",
serde_json::to_string(&json_array_results).unwrap_or_else(|_| "[]".to_string())
);
return Ok(());
}

if json_output {
let results = json!({
"summary": {
Expand Down Expand Up @@ -209,7 +246,10 @@ fn run_tests(
fn run_test_file(
file_path: &Path,
json_output: bool,
json_array: bool,
print_env_on_error: bool,
run_filter: Option<&Regex>,
json_array_results: &mut Vec<serde_json::Value>,
) -> Result<usize, Error> {
let content =
fs::read_to_string(file_path).map_err(|e| Error::FileRead(file_path.to_path_buf(), e))?;
Expand All @@ -220,23 +260,41 @@ fn run_test_file(
let mut test_count = 0;

for (test_name, test_case) in blockchain_test.0 {
if json_output {
// Output test start in JSON format
let output = json!({
"test": test_name,
"file": file_path.display().to_string(),
"status": "running"
});
print_json(&output);
} else {
println!(" Running: {test_name}");
if let Some(filter) = run_filter {
if !filter.is_match(&test_name) {
continue;
}
}
if !json_array {
if json_output {
// Output test start in JSON format
let output = json!({
"test": test_name,
"file": file_path.display().to_string(),
"status": "running"
});
print_json(&output);
} else {
println!(" Running: {test_name}");
}
}
// Execute the blockchain test
let result = execute_blockchain_test(&test_case, print_env_on_error, json_output);
let result = execute_blockchain_test(&test_case, print_env_on_error, json_output && !json_array);

// Get fork name for json_array output
let fork_name: &'static str = fork_to_spec_id(test_case.network).into();

match result {
Ok(()) => {
if json_output {
if json_array {
json_array_results.push(json!({
"name": test_name,
"pass": true,
"fork": fork_name,
"stateRoot": "",
"error": "",
}));
} else if json_output {
let output = json!({
"test": test_name,
"file": file_path.display().to_string(),
Expand All @@ -247,20 +305,30 @@ fn run_test_file(
test_count += 1;
}
Err(e) => {
if json_output {
let output = json!({
"test": test_name,
"file": file_path.display().to_string(),
"status": "failed",
"error": e.to_string()
if json_array {
json_array_results.push(json!({
"name": test_name,
"pass": false,
"fork": fork_name,
"stateRoot": "",
"error": e.to_string(),
}));
} else {
if json_output {
let output = json!({
"test": test_name,
"file": file_path.display().to_string(),
"status": "failed",
"error": e.to_string()
});
print_json(&output);
}
return Err(Error::TestExecution {
test_name,
test_path: file_path.to_path_buf(),
error: e.to_string(),
});
print_json(&output);
}
return Err(Error::TestExecution {
test_name,
test_path: file_path.to_path_buf(),
error: e.to_string(),
});
}
}
}
Expand Down Expand Up @@ -1226,4 +1294,7 @@ pub enum Error {

#[error("{failed} tests failed")]
TestsFailed { failed: usize },

#[error("Invalid regex: {0}")]
RegexError(String),
}
26 changes: 25 additions & 1 deletion bins/revme/src/cmd/statetest.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ pub use runner::{TestError as Error, TestErrorKind};

use crate::dir_utils::find_all_json_tests;
use clap::Parser;
use regex::Regex;
use runner::{run, TestError};
use std::path::PathBuf;

Expand Down Expand Up @@ -34,17 +35,36 @@ pub struct Cmd {
/// It will stop second run of EVM on failure.
#[arg(short = 'o', long)]
json_outcome: bool,
/// Output results as a JSON array with standard schema to stdout
///
/// Fields: name, pass, fork, stateRoot, error
#[arg(long)]
json_array: bool,
/// Omit progress output
#[arg(long)]
omit_progress: bool,
/// Keep going after a test failure
#[arg(long, alias = "no-fail-fast")]
keep_going: bool,
/// Only run tests whose name matches this regex
#[arg(long)]
run: Option<String>,
}

impl Cmd {
/// Runs `statetest` command.
pub fn run(&self) -> Result<(), TestError> {
let run_filter = self
.run
.as_deref()
.map(Regex::new)
.transpose()
.map_err(|e| TestError {
name: "Regex compilation".to_string(),
path: String::new(),
kind: TestErrorKind::RegexError(e.to_string()),
})?;

for path in &self.paths {
if !path.exists() {
return Err(TestError {
Expand All @@ -54,7 +74,9 @@ impl Cmd {
});
}

println!("\nRunning tests in {}...", path.display());
if !self.json_array {
println!("\nRunning tests in {}...", path.display());
}
let test_files = find_all_json_tests(path);

if test_files.is_empty() {
Expand All @@ -70,8 +92,10 @@ impl Cmd {
self.single_thread,
self.json,
self.json_outcome,
self.json_array,
self.keep_going,
self.omit_progress,
run_filter.clone(),
)?
}
Ok(())
Expand Down
Loading