diff --git a/tests/eest/eest_blockchain b/tests/eest/eest_blockchain new file mode 100755 index 0000000000..de2a32a573 Binary files /dev/null and b/tests/eest/eest_blockchain differ diff --git a/tests/eest/eest_blockchain.nim b/tests/eest/eest_blockchain.nim index 844541482f..15246ab706 100644 --- a/tests/eest/eest_blockchain.nim +++ b/tests/eest/eest_blockchain.nim @@ -17,8 +17,13 @@ import web3/execution_types, json_rpc/rpcclient, json_rpc/rpcserver, + ../../execution_chain/db/core_db/memory_only, ../../execution_chain/db/ledger, ../../execution_chain/core/chain/forked_chain, + ../../execution_chain/core/executor, + ../../execution_chain/core/validate, + ../../execution_chain/evm/state, + ../../execution_chain/evm/types, ../../execution_chain/beacon/beacon_engine, ../../execution_chain/common/common, ../../hive_integration/engine_client, @@ -26,6 +31,9 @@ import stew/byteutils, chronos +import ../../tools/common/helpers as chp except HardFork +import ../../tools/evmstate/helpers except HardFork + proc parseBlocks*(node: JsonNode): seq[BlockDesc] = for x in node: try: @@ -89,14 +97,278 @@ proc processFile*(fileName: string, statelessEnabled = false): bool = return testPass +proc runTestFast( + com: CommonRef, + parentHeader: Header, + baseTxFrame: CoreDbTxRef, + unit: BlockchainUnitEnv): Result[void, string] = + ## Execute blocks directly through the executor, bypassing ForkedChainRef. + let blocks = parseBlocks(unit.blocks) + var + parent = parentHeader + currentFrame = baseTxFrame + + for blk in blocks: + let childFrame = currentFrame.txFrameBegin() + + let vmState = BaseVMState() + vmState.init( + parent = parent, + header = blk.blk.header, + com = com, + txFrame = childFrame, + ) + + # Header + kinship validation (gas limits, timestamps, etc.) + let valRes = com.validateHeaderAndKinship( + blk.blk, + blockAccessList = Opt.none(BlockAccessListRef), + skipPreExecBalCheck = true, + parent, + childFrame) + if valRes.isErr: + if blk.badBlock: + childFrame.dispose() + continue + else: + childFrame.dispose() + return err("Good block failed validation: " & valRes.error) + + # Execute the block through the executor directly + let res = vmState.processBlock( + blk.blk, + skipValidation = false, + skipReceipts = false, + skipUncles = true, + skipStateRootCheck = false, + skipPostExecBalCheck = true, + ) + + if res.isOk: + if blk.badBlock: + childFrame.dispose() + return err("A bug? bad block imported") + # Persist the header so the next block can look up its parent + childFrame.persistHeader( + blk.blk.header.computeBlockHash, blk.blk.header).isOkOr: + childFrame.dispose() + return err("Failed to persist header: " & error) + parent = blk.blk.header + currentFrame = childFrame + else: + childFrame.dispose() + if not blk.badBlock: + return err("Good block rejected: " & res.error) + + # Verify final state root + let stateRoot = currentFrame.getStateRoot().valueOr: + return err("Failed to get state root") + if stateRoot != parent.stateRoot: + return err("Final stateRoot mismatch: got " & $stateRoot & + " expected " & $parent.stateRoot) + + ok() + +proc processFileFast*(fileName: string): bool = + let + fixture = parseFixture(fileName, BlockchainFixture) + + var testPass = true + for unit in fixture.units: + try: + let + header = unit.unit.genesisBlockHeader.to(Header) + memDB = newCoreDbRef DefaultDbMemory + baseTx = memDB.baseTxFrame() + ledger = LedgerRef.init(baseTx) + config = getChainConfig(unit.unit.network) + + config.chainId = unit.unit.config.chainid + config.blobSchedule = unit.unit.config.blobSchedule + + doAssert(unit.unit.genesisBlockHeader.hash == header.computeRlpHash) + + setupLedger(unit.unit.pre, ledger) + ledger.persist() + + baseTx.persistHeaderAndSetHead(header).isOkOr: + echo "\nTestName: ", unit.name, " Failed to persist genesis: ", error, "\n" + testPass = false + continue + + let com = CommonRef.new(memDB, config) + + let res = runTestFast(com, header, baseTx, unit.unit) + if res.isErr: + echo "\nTestName: ", unit.name, " RunTest error: ", res.error, "\n" + testPass = false + except ValueError as exc: + echo "\nTestName: ", unit.name, " Error: ", exc.msg, "\n" + testPass = false + + return testPass + when isMainModule: import - os, - unittest2 + std/[os, parseopt, strutils] + + type + TestResult = object + name: string + pass: bool + error: string - if paramCount() == 0: + proc collectJsonFiles(path: string): seq[string] = + if fileExists(path): + return @[path] + for entry in walkDirRec(path): + if entry.endsWith(".json") and "/.meta/" notin entry: + result.add(entry) + + proc printUsage() = let testFile = getAppFilename().splitPath().tail - echo "Usage: " & testFile & " vector.json" + echo "Usage: " & testFile & " [options] " + echo "" + echo "Options:" + echo " --fast Bypass ForkedChainRef; call executor directly" + echo " --run= Substring filter on file paths" + echo " --json Output results as JSON array" + echo " --workers= Number of workers (accepted, runs sequentially)" + echo "" + echo "Examples:" + echo " " & testFile & " vector.json" + echo " " & testFile & " --fast /path/to/blockchain_tests/" + echo " " & testFile & " --json /path/to/blockchain_tests/" + echo " " & testFile & " --run=eip7702 /path/to/blockchain_tests/" + + var + fastEnabled = false + jsonEnabled = false + runFilter = "" + workers = 1 + inputPath = "" + + var p = initOptParser(commandLineParams()) + while true: + p.next() + case p.kind + of cmdEnd: break + of cmdLongOption, cmdShortOption: + case p.key.toLowerAscii + of "fast": + fastEnabled = true + of "json": + jsonEnabled = true + of "run": + runFilter = p.val + of "workers": + workers = parseInt(p.val) + of "help", "h": + printUsage() + quit(QuitSuccess) + else: + echo "Unknown option: ", p.key + printUsage() + quit(QuitFailure) + of cmdArgument: + inputPath = p.key + + if inputPath.len == 0: + printUsage() + quit(QuitFailure) + + var files = collectJsonFiles(inputPath) + + if runFilter.len > 0: + var filtered: seq[string] + for f in files: + if runFilter in f: + filtered.add(f) + files = filtered + + if files.len == 0: + echo "No matching .json files found." quit(QuitFailure) - check processFile(paramStr(1)) + type FileResult = object + path: string + pass: bool + + var + results: seq[TestResult] + passCount = 0 + failCount = 0 + + if workers > 1: + discard workers # TODO: in-process parallelism requires GC-safe procs + + for f in files: + if jsonEnabled: + # Per-test results for JSON output + let fixture = parseFixture(f, BlockchainFixture) + for unit in fixture.units: + let header = unit.unit.genesisBlockHeader.to(Header) + if fastEnabled: + try: + let + memDB = newCoreDbRef DefaultDbMemory + baseTx = memDB.baseTxFrame() + ledger = LedgerRef.init(baseTx) + config = getChainConfig(unit.unit.network) + config.chainId = unit.unit.config.chainid + config.blobSchedule = unit.unit.config.blobSchedule + setupLedger(unit.unit.pre, ledger) + ledger.persist() + let persistRes = baseTx.persistHeaderAndSetHead(header) + if persistRes.isErr: + inc failCount + results.add(TestResult(name: unit.name, pass: false, + error: "persist genesis: " & persistRes.error)) + continue + let com = CommonRef.new(memDB, config) + let res = runTestFast(com, header, baseTx, unit.unit) + if res.isOk: + inc passCount + results.add(TestResult(name: unit.name, pass: true, error: "")) + else: + inc failCount + results.add(TestResult(name: unit.name, pass: false, error: res.error)) + except CatchableError as e: + inc failCount + results.add(TestResult(name: unit.name, pass: false, error: e.msg)) + else: + let env = prepareEnv(unit.unit, header, rpcEnabled = false) + let res = waitFor env.runTest(unit.unit) + if res.isOk: + inc passCount + results.add(TestResult(name: unit.name, pass: true, error: "")) + else: + inc failCount + results.add(TestResult(name: unit.name, pass: false, error: res.error)) + env.close() + else: + let pass = if fastEnabled: processFileFast(f) + else: processFile(f) + let rel = if dirExists(inputPath): + f.relativePath(inputPath) + else: + f.splitPath().tail + if pass: + inc passCount + echo "PASS: ", rel + else: + inc failCount + echo "FAIL: ", rel + + if jsonEnabled: + var arr = newJArray() + for r in results: + arr.add(%*{"name": r.name, "pass": r.pass, "error": r.error}) + echo $arr + else: + echo "" + echo "Total: ", files.len, " | Passed: ", passCount, + " | Failed: ", failCount + + if failCount > 0: + quit(QuitFailure) diff --git a/tests/eest/eest_engine b/tests/eest/eest_engine new file mode 100755 index 0000000000..24d9b1c1c5 Binary files /dev/null and b/tests/eest/eest_engine differ diff --git a/tests/eest/eest_engine.nim b/tests/eest/eest_engine.nim index a612809812..b49317b04b 100644 --- a/tests/eest/eest_engine.nim +++ b/tests/eest/eest_engine.nim @@ -18,10 +18,12 @@ import web3/execution_types, json_rpc/rpcclient, json_rpc/rpcserver, + chronos, ../../execution_chain/db/ledger, ../../execution_chain/core/chain/forked_chain, ../../execution_chain/core/tx_pool, ../../execution_chain/beacon/beacon_engine, + ../../execution_chain/beacon/api_handler, ../../execution_chain/common/common, ../../hive_integration/engine_client, ./eest_helpers @@ -128,14 +130,225 @@ proc processFile*(fileName: string, statelessEnabled = false): bool = return testPass +proc toVersion(n: uint64): Version = + ## Convert 1-based Numero to Version enum. + case n + of 1: Version.V1 + of 2: Version.V2 + of 3: Version.V3 + of 4: Version.V4 + of 5: Version.V5 + else: Version.V1 + +proc sendNewPayloadDirect(ben: BeaconEngineRef, version: uint64, param: PayloadParam): Result[PayloadStatusV1, string] = + let npVersion = toVersion(version) + try: + ok(waitFor ben.newPayload( + npVersion, + param.payload, + param.versionedHashes, + param.parentBeaconBlockRoot, + param.executionRequests)) + except ApplicationError as exc: + err(exc.msg) + except RlpError as exc: + err(exc.msg) + except CancelledError: + err("cancelled") + +proc sendFCUDirect(ben: BeaconEngineRef, version: uint64, param: PayloadParam): Result[ForkchoiceUpdatedResponse, string] = + let + fcuVersion = toVersion(version) + update = ForkchoiceStateV1( + headblockHash: param.payload.blockHash, + finalizedblockHash: param.payload.blockHash + ) + try: + ok(waitFor ben.forkchoiceUpdated( + fcuVersion, update, Opt.none(PayloadAttributes))) + except ApplicationError as exc: + err(exc.msg) + except CancelledError: + err("cancelled") + +proc runTestFast(env: TestEnv, unit: EngineUnitEnv): Result[void, string] = + ## Execute engine payloads via direct BeaconEngine method calls, + ## bypassing the HTTP RPC layer. + let ben = env.beaconEngine.get() + + for enp in unit.engineNewPayloads: + var status = ben.sendNewPayloadDirect(enp.newPayloadVersion.uint64, enp.params).valueOr: + if enp.validationError.isSome(): + continue + else: + return err(error) + + discard status + + let y = ben.sendFCUDirect(enp.forkchoiceUpdatedVersion.uint64, enp.params).valueOr: + return err(error) + + discard y + + let header = env.chain.latestHeader() + + if unit.lastblockhash != header.computeRlpHash: + return err("last block hash mismatch") + + ok() + +proc processFileFast*(fileName: string): bool = + let + fixture = parseFixture(fileName, EngineFixture) + + var testPass = true + for unit in fixture.units: + let header = unit.unit.genesisBlockHeader.to(Header) + doAssert(unit.unit.genesisBlockHeader.hash == header.computeRlpHash) + let env = prepareEnv(unit.unit, header, engineDirect = true) + env.runTestFast(unit.unit).isOkOr: + echo "\nTestName: ", unit.name, " RunTest error: ", error, "\n" + testPass = false + env.close() + + return testPass + +{.pop.} # undo {.push raises: [], gcsafe.} for isMainModule block + when isMainModule: import - std/[cmdline, os], - unittest2 + std/[json, os, parseopt, strutils] - if paramCount() == 0: + type + TestResult = object + name: string + pass: bool + error: string + + proc collectJsonFiles(path: string): seq[string] {.raises: [OSError].} = + if fileExists(path): + return @[path] + for entry in walkDirRec(path): + if entry.endsWith(".json") and "/.meta/" notin entry: + result.add(entry) + + proc printUsage() = let testFile = getAppFilename().splitPath().tail - echo "Usage: " & testFile & " vector.json" + echo "Usage: " & testFile & " [options] " + echo "" + echo "Options:" + echo " --fast Bypass HTTP RPC; call BeaconEngine directly" + echo " --run= Substring filter on file paths" + echo " --json Output results as JSON array" + echo " --workers= Number of workers (accepted, runs sequentially)" + echo "" + echo "Examples:" + echo " " & testFile & " vector.json" + echo " " & testFile & " --fast /path/to/blockchain_tests_engine/" + echo " " & testFile & " --json /path/to/blockchain_tests_engine/" + echo " " & testFile & " --run=eip7702 /path/to/blockchain_tests_engine/" + + var + fastEnabled = false + jsonEnabled = false + runFilter = "" + workers = 1 + inputPath = "" + + var p = initOptParser(commandLineParams()) + while true: + p.next() + case p.kind + of cmdEnd: break + of cmdLongOption, cmdShortOption: + case p.key.toLowerAscii + of "fast": + fastEnabled = true + of "json": + jsonEnabled = true + of "run": + runFilter = p.val + of "workers": + workers = parseInt(p.val) + of "help", "h": + printUsage() + quit(QuitSuccess) + else: + echo "Unknown option: ", p.key + printUsage() + quit(QuitFailure) + of cmdArgument: + inputPath = p.key + + discard workers # sequential only; flag accepted for CLI compatibility + + if inputPath.len == 0: + printUsage() + quit(QuitFailure) + + var files = collectJsonFiles(inputPath) + + if runFilter.len > 0: + var filtered: seq[string] + for f in files: + if runFilter in f: + filtered.add(f) + files = filtered + + if files.len == 0: + echo "No matching .json files found." quit(QuitFailure) - check processFile(paramStr(1)) + var + results: seq[TestResult] + passCount = 0 + failCount = 0 + + for f in files: + if jsonEnabled: + # Per-test results for JSON output + let fixture = parseFixture(f, EngineFixture) + for unit in fixture.units: + let header = unit.unit.genesisBlockHeader.to(Header) + let env = if fastEnabled: + prepareEnv(unit.unit, header, engineDirect = true) + else: + prepareEnv(unit.unit, header, rpcEnabled = true) + let res = if fastEnabled: + env.runTestFast(unit.unit) + else: + env.runTest(unit.unit) + var errMsg = "" + if res.isOk: + inc passCount + else: + errMsg = res.error + inc failCount + results.add(TestResult(name: unit.name, pass: res.isOk, error: errMsg)) + env.close() + else: + let pass = if fastEnabled: processFileFast(f) + else: processFile(f) + let rel = if dirExists(inputPath): + f.relativePath(inputPath) + else: + f.splitPath().tail + if pass: + inc passCount + echo "PASS: ", rel + else: + inc failCount + echo "FAIL: ", rel + + if jsonEnabled: + var arr = newJArray() + for r in results: + arr.add(%*{"name": r.name, "pass": r.pass, "error": r.error}) + echo $arr + else: + echo "" + echo "Total: ", files.len, " | Passed: ", passCount, + " | Failed: ", failCount + + if failCount > 0: + quit(QuitFailure) diff --git a/tests/eest/eest_helpers.nim b/tests/eest/eest_helpers.nim index 2302b2168d..0ae819d8a9 100644 --- a/tests/eest/eest_helpers.nim +++ b/tests/eest/eest_helpers.nim @@ -94,6 +94,7 @@ type TestEnv* = ref object chain*: ForkedChainRef + beaconEngine*: Opt[BeaconEngineRef] server*: Opt[RpcHttpServer] client*: Opt[RpcHttpClient] @@ -223,6 +224,7 @@ proc prepareEnv*( unit: UnitEnv, genesis: Header, rpcEnabled = false, + engineDirect = false, statelessEnabled = false): TestEnv = try: @@ -250,6 +252,7 @@ proc prepareEnv*( chain = ForkedChainRef.init(com, enableQueue = true, persistBatchSize = 1) testEnv.chain = chain + testEnv.beaconEngine = Opt.none(BeaconEngineRef) testEnv.client = Opt.none(RpcHttpClient) testEnv.server = Opt.none(RpcHttpServer) @@ -269,8 +272,14 @@ proc prepareEnv*( let client = setupClient(server.localAddress[0].port) + testEnv.beaconEngine = Opt.some(beaconEngine) testEnv.client = Opt.some(client) testEnv.server = Opt.some(server) + elif engineDirect: + let + txPool = TxPoolRef.new(chain) + beaconEngine = BeaconEngineRef.new(txPool) + testEnv.beaconEngine = Opt.some(beaconEngine) return testEnv except ValueError as exc: diff --git a/tools/evmstate/config.nim b/tools/evmstate/config.nim index 9b3ede8c69..7532ab5063 100644 --- a/tools/evmstate/config.nim +++ b/tools/evmstate/config.nim @@ -74,8 +74,18 @@ type defaultValue: 0 name: "verbosity" }: int + run* {. + desc: "regex filter for test names" + defaultValue: "" + name: "run" }: string + + workers* {. + desc: "number of parallel workers (0 = auto)" + defaultValue: 1 + name: "workers" }: int + inputFile* {. - desc: "json file contains state test data" + desc: "json file or directory containing state test data" defaultValue: "" argument }: string diff --git a/tools/evmstate/evmstate b/tools/evmstate/evmstate new file mode 100755 index 0000000000..9a60449cbe Binary files /dev/null and b/tools/evmstate/evmstate differ diff --git a/tools/evmstate/evmstate.nim b/tools/evmstate/evmstate.nim index f0f7060c89..e4d372207a 100644 --- a/tools/evmstate/evmstate.nim +++ b/tools/evmstate/evmstate.nim @@ -9,7 +9,7 @@ # according to those terms. import - std/[json, strutils, sets, tables, options, streams], + std/[json, strutils, sets, tables, options, streams, os], chronicles, eth/common/keys, eth/common/transaction_utils, @@ -170,7 +170,8 @@ proc toTracerFlags(conf: StateConf): set[TracerFlags] = template hasError(ctx: StateContext): bool = ctx.error.len > 0 -proc prepareAndRun(inputFile: string, conf: StateConf): bool = +proc prepareAndRun(inputFile: string, conf: StateConf, + allResults: var seq[StateResult]): bool = var ctx: StateContext @@ -188,7 +189,6 @@ proc prepareAndRun(inputFile: string, conf: StateConf): bool = ctx.tracerFlags = toTracerFlags(conf) var - stateRes = newSeqOfCap[StateResult](post.len) index = 1 hasError = false @@ -207,7 +207,7 @@ proc prepareAndRun(inputFile: string, conf: StateConf): bool = ctx.expectedLogs = Hash32.fromJson(subTest["logs"]) ctx.tx = parseTx(txData, subTest["indexes"]) let res = ctx.runExecution(conf, pre) - stateRes.add res + allResults.add res hasError = hasError or ctx.hasError if conf.fork.len > 0: @@ -235,7 +235,6 @@ proc prepareAndRun(inputFile: string, conf: StateConf): bool = for subTest in forkData: runSubTest(subTest) - writeResultToStdout(stateRes) not hasError when defined(chronicles_runtime_filtering): @@ -253,6 +252,15 @@ when defined(chronicles_runtime_filtering): let level = v.toLogLevel setLogLevel(level) +proc collectJsonFiles(path: string): seq[string] = + ## Recursively collect all .json files under a directory, or return + ## the single file if path is a file. + if fileExists(path): + return @[path] + for entry in walkDirRec(path): + if entry.endsWith(".json") and "/.meta/" notin entry: + result.add(entry) + proc main() = # https://github.com/status-im/nimbus-eth1/issues/3131 setStdIoUnbuffered() @@ -261,14 +269,41 @@ proc main() = when defined(chronicles_runtime_filtering): setVerbosity(conf.verbosity) + let hasFilter = conf.run.len > 0 + var allResults: seq[StateResult] + if conf.inputFile.len > 0: - if not prepareAndRun(conf.inputFile, conf): - quit(QuitFailure) + if dirExists(conf.inputFile): + # Directory mode: collect all JSON files and run sequentially + var files = collectJsonFiles(conf.inputFile) + if hasFilter: + var filtered: seq[string] + for f in files: + if conf.run in f: + filtered.add(f) + files = filtered + + var noError = true + for f in files: + let res = prepareAndRun(f, conf, allResults) + noError = noError and res + + writeResultToStdout(allResults) + if not noError: + quit(QuitFailure) + else: + # Single file mode + if not prepareAndRun(conf.inputFile, conf, allResults): + writeResultToStdout(allResults) + quit(QuitFailure) + writeResultToStdout(allResults) else: + # Stdin mode var noError = true for inputFile in lines(stdin): - let res = prepareAndRun(inputFile, conf) + let res = prepareAndRun(inputFile, conf, allResults) noError = noError and res + writeResultToStdout(allResults) if not noError: quit(QuitFailure)