diff --git a/Cargo.lock b/Cargo.lock index c3700c70dd..c49bf8b0fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3787,6 +3787,7 @@ dependencies = [ "csv", "indicatif", "k256", + "regex", "revm", "serde", "serde_json", diff --git a/bins/revme/Cargo.toml b/bins/revme/Cargo.toml index acafd2c03c..3e9167922d 100644 --- a/bins/revme/Cargo.toml +++ b/bins/revme/Cargo.toml @@ -38,6 +38,7 @@ thiserror.workspace = true walkdir.workspace = true k256 = { workspace = true, features = ["ecdsa"] } csv = "1.1.6" +regex = "1" [features] default = ["map-foldhash"] diff --git a/bins/revme/src/cmd.rs b/bins/revme/src/cmd.rs index 2043f92db0..4c1faa1487 100644 --- a/bins/revme/src/cmd.rs +++ b/bins/revme/src/cmd.rs @@ -7,7 +7,7 @@ pub mod statetest; use clap::Parser; #[derive(Parser, Debug)] -#[command(infer_subcommands = true)] +#[command(infer_subcommands = true, version)] #[allow(clippy::large_enum_variant)] pub enum MainCmd { /// Execute Ethereum state tests. diff --git a/bins/revme/src/cmd/blockchaintest.rs b/bins/revme/src/cmd/blockchaintest.rs index 7c38ac0da1..99e6481f0b 100644 --- a/bins/revme/src/cmd/blockchaintest.rs +++ b/bins/revme/src/cmd/blockchaintest.rs @@ -3,6 +3,7 @@ pub mod pre_block; use crate::dir_utils::find_all_json_tests; use clap::Parser; +use regex::Regex; use revm::statetest_types::blockchain::{ Account, BlockchainTest, BlockchainTestCase, ForkSpec, Withdrawal, @@ -55,11 +56,26 @@ pub struct Cmd { /// Output results in JSON format #[arg(long)] json: bool, + /// Output results as a JSON array with standard schema to stdout + /// + /// Fields: name, pass, fork, stateRoot, error + #[arg(long)] + json_array: bool, + /// Only run tests whose name matches this regex + #[arg(long)] + run: Option, } impl Cmd { /// Runs `blockchaintest` command. pub fn run(&self) -> Result<(), Error> { + let run_filter = self + .run + .as_deref() + .map(Regex::new) + .transpose() + .map_err(|e| Error::RegexError(e.to_string()))?; + for path in &self.paths { if !path.exists() { return Err(Error::PathNotFound(path.clone())); @@ -80,6 +96,8 @@ impl Cmd { self.keep_going, self.print_env_on_error, self.json, + self.json_array, + run_filter.as_ref(), )?; } Ok(()) @@ -93,11 +111,15 @@ fn run_tests( keep_going: bool, print_env_on_error: bool, json_output: bool, + json_array: bool, + run_filter: Option<&Regex>, ) -> Result<(), Error> { let mut passed = 0; let mut failed = 0; let mut skipped = 0; let mut failed_paths = Vec::new(); + let mut json_array_results: Vec = Vec::new(); + let keep_going = keep_going || json_array; let start_time = Instant::now(); let total_files = test_files.len(); @@ -124,7 +146,14 @@ fn run_tests( continue; } - let result = run_test_file(&file_path, json_output, print_env_on_error); + let result = run_test_file( + &file_path, + json_output, + json_array, + print_env_on_error, + run_filter, + &mut json_array_results, + ); match result { Ok(test_count) => { @@ -172,6 +201,14 @@ fn run_tests( let duration = start_time.elapsed(); + if json_array { + println!( + "{}", + serde_json::to_string(&json_array_results).unwrap_or_else(|_| "[]".to_string()) + ); + return Ok(()); + } + if json_output { let results = json!({ "summary": { @@ -209,7 +246,10 @@ fn run_tests( fn run_test_file( file_path: &Path, json_output: bool, + json_array: bool, print_env_on_error: bool, + run_filter: Option<&Regex>, + json_array_results: &mut Vec, ) -> Result { let content = fs::read_to_string(file_path).map_err(|e| Error::FileRead(file_path.to_path_buf(), e))?; @@ -220,23 +260,41 @@ fn run_test_file( let mut test_count = 0; for (test_name, test_case) in blockchain_test.0 { - if json_output { - // Output test start in JSON format - let output = json!({ - "test": test_name, - "file": file_path.display().to_string(), - "status": "running" - }); - print_json(&output); - } else { - println!(" Running: {test_name}"); + if let Some(filter) = run_filter { + if !filter.is_match(&test_name) { + continue; + } + } + if !json_array { + if json_output { + // Output test start in JSON format + let output = json!({ + "test": test_name, + "file": file_path.display().to_string(), + "status": "running" + }); + print_json(&output); + } else { + println!(" Running: {test_name}"); + } } // Execute the blockchain test - let result = execute_blockchain_test(&test_case, print_env_on_error, json_output); + let result = execute_blockchain_test(&test_case, print_env_on_error, json_output && !json_array); + + // Get fork name for json_array output + let fork_name: &'static str = fork_to_spec_id(test_case.network).into(); match result { Ok(()) => { - if json_output { + if json_array { + json_array_results.push(json!({ + "name": test_name, + "pass": true, + "fork": fork_name, + "stateRoot": "", + "error": "", + })); + } else if json_output { let output = json!({ "test": test_name, "file": file_path.display().to_string(), @@ -247,20 +305,30 @@ fn run_test_file( test_count += 1; } Err(e) => { - if json_output { - let output = json!({ - "test": test_name, - "file": file_path.display().to_string(), - "status": "failed", - "error": e.to_string() + if json_array { + json_array_results.push(json!({ + "name": test_name, + "pass": false, + "fork": fork_name, + "stateRoot": "", + "error": e.to_string(), + })); + } else { + if json_output { + let output = json!({ + "test": test_name, + "file": file_path.display().to_string(), + "status": "failed", + "error": e.to_string() + }); + print_json(&output); + } + return Err(Error::TestExecution { + test_name, + test_path: file_path.to_path_buf(), + error: e.to_string(), }); - print_json(&output); } - return Err(Error::TestExecution { - test_name, - test_path: file_path.to_path_buf(), - error: e.to_string(), - }); } } } @@ -1226,4 +1294,7 @@ pub enum Error { #[error("{failed} tests failed")] TestsFailed { failed: usize }, + + #[error("Invalid regex: {0}")] + RegexError(String), } diff --git a/bins/revme/src/cmd/statetest.rs b/bins/revme/src/cmd/statetest.rs index dca60accee..5c4c10f800 100644 --- a/bins/revme/src/cmd/statetest.rs +++ b/bins/revme/src/cmd/statetest.rs @@ -6,6 +6,7 @@ pub use runner::{TestError as Error, TestErrorKind}; use crate::dir_utils::find_all_json_tests; use clap::Parser; +use regex::Regex; use runner::{run, TestError}; use std::path::PathBuf; @@ -34,17 +35,36 @@ pub struct Cmd { /// It will stop second run of EVM on failure. #[arg(short = 'o', long)] json_outcome: bool, + /// Output results as a JSON array with standard schema to stdout + /// + /// Fields: name, pass, fork, stateRoot, error + #[arg(long)] + json_array: bool, /// Omit progress output #[arg(long)] omit_progress: bool, /// Keep going after a test failure #[arg(long, alias = "no-fail-fast")] keep_going: bool, + /// Only run tests whose name matches this regex + #[arg(long)] + run: Option, } impl Cmd { /// Runs `statetest` command. pub fn run(&self) -> Result<(), TestError> { + let run_filter = self + .run + .as_deref() + .map(Regex::new) + .transpose() + .map_err(|e| TestError { + name: "Regex compilation".to_string(), + path: String::new(), + kind: TestErrorKind::RegexError(e.to_string()), + })?; + for path in &self.paths { if !path.exists() { return Err(TestError { @@ -54,7 +74,9 @@ impl Cmd { }); } - println!("\nRunning tests in {}...", path.display()); + if !self.json_array { + println!("\nRunning tests in {}...", path.display()); + } let test_files = find_all_json_tests(path); if test_files.is_empty() { @@ -70,8 +92,10 @@ impl Cmd { self.single_thread, self.json, self.json_outcome, + self.json_array, self.keep_going, self.omit_progress, + run_filter.clone(), )? } Ok(()) diff --git a/bins/revme/src/cmd/statetest/runner.rs b/bins/revme/src/cmd/statetest/runner.rs index 8c72fff2a2..c4de6c15c4 100644 --- a/bins/revme/src/cmd/statetest/runner.rs +++ b/bins/revme/src/cmd/statetest/runner.rs @@ -10,6 +10,7 @@ use revm::{ statetest_types::{SpecName, Test, TestSuite, TestUnit}, Context, ExecuteCommitEvm, InspectEvm, MainBuilder, MainContext, }; +use regex::Regex; use serde_json::json; use std::{ convert::Infallible, @@ -60,6 +61,8 @@ pub enum TestErrorKind { InvalidPath, #[error("no JSON test files found in path")] NoJsonFiles, + #[error("invalid regex: {0}")] + RegexError(String), } /// Check if a test should be skipped based on its filename @@ -115,6 +118,7 @@ struct TestExecutionContext<'a> { elapsed: &'a Arc>, trace: bool, print_json_outcome: bool, + json_array_results: Option<&'a Arc>>>, } struct DebugContext<'a> { @@ -216,9 +220,26 @@ fn check_evm_execution( db: &mut database::State, spec: SpecId, print_json_outcome: bool, + json_array_results: Option<&Arc>>>, ) -> Result<(), TestErrorKind> { let validation = compute_test_roots(exec_result, db); + let collect_json_array = |error: Option<&TestErrorKind>| { + if let Some(results) = json_array_results { + let fork_name: &'static str = spec.into(); + let state_root = format!("{}", validation.state_root); + let error_str = error.map(|e| e.to_string()).unwrap_or_default(); + let entry = json!({ + "name": test_name, + "pass": error.is_none(), + "fork": fork_name, + "stateRoot": state_root, + "error": error_str, + }); + results.lock().unwrap().push(entry); + } + }; + let print_json = |error: Option<&TestErrorKind>| { if print_json_outcome { let json = build_json_output( @@ -235,11 +256,13 @@ fn check_evm_execution( // Check if exception handling is correct let exception_expected = validate_exception(test, exec_result).inspect_err(|e| { + collect_json_array(Some(e)); print_json(Some(e)); })?; // If exception was expected and occurred, we're done if exception_expected { + collect_json_array(None); print_json(None); return Ok(()); } @@ -247,6 +270,7 @@ fn check_evm_execution( // Validate output if execution succeeded if let Ok(result) = exec_result { validate_output(expected_output, result).inspect_err(|e| { + collect_json_array(Some(e)); print_json(Some(e)); })?; } @@ -257,6 +281,7 @@ fn check_evm_execution( got: validation.logs_root, expected: test.logs, }; + collect_json_array(Some(&error)); print_json(Some(&error)); return Err(error); } @@ -267,10 +292,12 @@ fn check_evm_execution( got: validation.state_root, expected: test.hash, }; + collect_json_array(Some(&error)); print_json(Some(&error)); return Err(error); } + collect_json_array(None); print_json(None); Ok(()) } @@ -287,6 +314,8 @@ pub fn execute_test_suite( elapsed: &Arc>, trace: bool, print_json_outcome: bool, + json_array_results: Option<&Arc>>>, + run_filter: Option<&Regex>, ) -> Result<(), TestError> { if skip_test(path) { return Ok(()); @@ -301,6 +330,11 @@ pub fn execute_test_suite( })?; for (name, unit) in suite.0 { + if let Some(filter) = run_filter { + if !filter.is_match(&name) { + continue; + } + } // Prepare initial state let cache_state = unit.state(); @@ -360,6 +394,7 @@ pub fn execute_test_suite( elapsed, trace, print_json_outcome, + json_array_results, }); if let Err(e) = result { @@ -438,6 +473,7 @@ fn execute_single_test(ctx: TestExecutionContext) -> Result<(), TestErrorKind> { db, *ctx.cfg.spec(), ctx.print_json_outcome, + ctx.json_array_results, ) } @@ -480,16 +516,25 @@ fn debug_failed_test(ctx: DebugContext) { ); } -#[derive(Clone, Copy)] +#[derive(Clone)] struct TestRunnerConfig { single_thread: bool, trace: bool, print_outcome: bool, keep_going: bool, + json_array: bool, + run_filter: Option, } impl TestRunnerConfig { - fn new(single_thread: bool, trace: bool, print_outcome: bool, keep_going: bool) -> Self { + fn new( + single_thread: bool, + trace: bool, + print_outcome: bool, + json_array: bool, + keep_going: bool, + run_filter: Option, + ) -> Self { // Trace implies print_outcome let print_outcome = print_outcome || trace; // print_outcome or trace implies single_thread @@ -500,6 +545,8 @@ impl TestRunnerConfig { trace, print_outcome, keep_going, + json_array, + run_filter, } } } @@ -511,12 +558,13 @@ struct TestRunnerState { queue: Arc)>>, elapsed: Arc>, errors: Arc>>, + json_array_results: Option>>>, } impl TestRunnerState { - fn new(test_files: Vec, omit_progress: bool) -> Self { + fn new(test_files: Vec, omit_progress: bool, json_array: bool) -> Self { let n_files = test_files.len(); - let draw_target = if omit_progress { + let draw_target = if omit_progress || json_array { ProgressDrawTarget::hidden() } else { ProgressDrawTarget::stdout() @@ -530,6 +578,11 @@ impl TestRunnerState { queue: Arc::new(Mutex::new((0usize, test_files))), elapsed: Arc::new(Mutex::new(Duration::ZERO)), errors: Arc::new(Mutex::new(Vec::new())), + json_array_results: if json_array { + Some(Arc::new(Mutex::new(Vec::new()))) + } else { + None + }, } } @@ -557,6 +610,8 @@ fn run_test_worker(state: TestRunnerState, config: TestRunnerConfig) -> Result<( &state.elapsed, config.trace, config.print_outcome, + state.json_array_results.as_ref(), + config.run_filter.as_ref(), ); state.console_bar.inc(1); @@ -592,18 +647,22 @@ pub fn run( single_thread: bool, trace: bool, print_outcome: bool, + json_array: bool, keep_going: bool, omit_progress: bool, + run_filter: Option, ) -> Result<(), TestError> { - let config = TestRunnerConfig::new(single_thread, trace, print_outcome, keep_going); + let keep_going = keep_going || json_array; + let config = TestRunnerConfig::new(single_thread, trace, print_outcome, json_array, keep_going, run_filter); let n_files = test_files.len(); - let state = TestRunnerState::new(test_files, omit_progress); + let state = TestRunnerState::new(test_files, omit_progress, json_array); let num_threads = determine_thread_count(config.single_thread, n_files); // Spawn worker threads let mut handles = Vec::with_capacity(num_threads); for i in 0..num_threads { let state = state.clone(); + let config = config.clone(); let thread = std::thread::Builder::new() .name(format!("runner-{i}")) @@ -629,6 +688,13 @@ pub fn run( state.console_bar.finish(); + // Output JSON array if requested + if let Some(results) = &state.json_array_results { + let results = results.lock().unwrap(); + println!("{}", serde_json::to_string(&*results).unwrap_or_else(|_| "[]".to_string())); + return Ok(()); + } + // Print summary println!( "Finished execution. Total CPU time: {:.6}s",