From 671ee6429ecf3d9371b6c5113c407232861f80d9 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 8 May 2026 11:20:02 -0700 Subject: [PATCH 1/3] Add MAUI Desktop BenchmarkDotNet benchmarks scenario Adds a new Helix work item that clones dotnet/maui at the channel's matching branch, patches the project tree to disable mobile target frameworks, injects PerfLabExporter via BenchmarkDotNet.Extensions, and runs the Core, XAML, and Graphics BDN suites. Public CI runs on win-x64 (channel main); private internal CI runs on win-x64-viper. Layout: - eng/performance/maui_desktop_benchmarks.proj Helix work item definition (TargetsWindows-gated). - eng/pipelines/sdk-perf-jobs.yml Two new build-matrix entries (public + private) using the existing run-scenarios-job.yml template. - src/scenarios/mauiDesktopBenchmarks/{pre,test,post}.py Scenario payload. pre.py runs on the AzDO agent (just logs the framework), test.py runs on Helix (clones MAUI, patches, builds, runs BDN suites), post.py cleans up the cloned repo. - src/scenarios/shared/bdndesktop.py Reusable BDNDesktopHelper with the generic external-repo BDN workflow (patch Directory.Build.props, inject BDN.Extensions, build, run, collect/upload reports). Designed so other repos (e.g. dotnet/runtime libraries) can drop in with just a config dict. - src/harness/BenchmarkDotNet.Extensions/ExclusionFilter.cs Made public so the patched-in Program.cs in external benchmarks can construct it. Notable design points: - test.py acquires source via git sparse-checkout when git is on PATH, falling back to a curl-based zip download (Windows cert store) when it is not (Helix machines without git installed). - patch_directory_build_props regex-replaces Include*TargetFrameworks in place rather than appending overrides, because MAUI sets these properties at multiple points and computes MauiPlatforms from them; appending at the end has no effect. - _inject_bdn_extensions is idempotent: it removes any prior ItemGroup labeled PerfLabInjected before adding a new one, so re-running against an existing checkout doesn't accumulate duplicate ProjectReferences. - _patch_program_cs raises if the BenchmarkSwitcher.Run pattern isn't found, preventing silent runs that produce no perf-lab-report.json. - BDNDesktopHelper exposes bdn_run_params and strict_build kwargs so callers can tune iterations or require all suites to build. - parse_args() uses parse_known_args() so unknown flags (--filter, --maxIterationCount, etc.) flow through to BDN. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/performance/maui_desktop_benchmarks.proj | 19 + eng/pipelines/sdk-perf-jobs.yml | 30 ++ .../ExclusionFilter.cs | 2 +- src/scenarios/mauiDesktopBenchmarks/post.py | 33 ++ src/scenarios/mauiDesktopBenchmarks/pre.py | 19 + src/scenarios/mauiDesktopBenchmarks/test.py | 296 ++++++++++++ src/scenarios/shared/bdndesktop.py | 445 ++++++++++++++++++ 7 files changed, 843 insertions(+), 1 deletion(-) create mode 100644 eng/performance/maui_desktop_benchmarks.proj create mode 100644 src/scenarios/mauiDesktopBenchmarks/post.py create mode 100644 src/scenarios/mauiDesktopBenchmarks/pre.py create mode 100644 src/scenarios/mauiDesktopBenchmarks/test.py create mode 100644 src/scenarios/shared/bdndesktop.py diff --git a/eng/performance/maui_desktop_benchmarks.proj b/eng/performance/maui_desktop_benchmarks.proj new file mode 100644 index 00000000000..047a4f9f9b5 --- /dev/null +++ b/eng/performance/maui_desktop_benchmarks.proj @@ -0,0 +1,19 @@ + + + + + + + + + $(ScenariosDir)mauiDesktopBenchmarks + $(Python) pre.py -f $(PERFLAB_Framework) + $(Python) test.py --framework $(PERFLAB_Framework) --suite all + $(Python) post.py + + + + + diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index c3c87fca015..45091e45509 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -93,6 +93,21 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} + # MAUI Desktop BenchmarkDotNet benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + isPublic: true + jobParameters: + runKind: maui_desktop_benchmarks + projectFileName: maui_desktop_benchmarks.proj + channels: + - main + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + # Blazor scenario benchmarks - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: @@ -623,6 +638,21 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} + # MAUI Desktop BDN benchmarks (private) + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + isPublic: false + jobParameters: + runKind: maui_desktop_benchmarks + projectFileName: maui_desktop_benchmarks.proj + channels: + - main + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + # NativeAOT scenario benchmarks - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: diff --git a/src/harness/BenchmarkDotNet.Extensions/ExclusionFilter.cs b/src/harness/BenchmarkDotNet.Extensions/ExclusionFilter.cs index c60d3cb1db4..4d1cd029c24 100644 --- a/src/harness/BenchmarkDotNet.Extensions/ExclusionFilter.cs +++ b/src/harness/BenchmarkDotNet.Extensions/ExclusionFilter.cs @@ -6,7 +6,7 @@ namespace BenchmarkDotNet.Extensions { - class ExclusionFilter : IFilter + public class ExclusionFilter : IFilter { private readonly GlobFilter? globFilter; diff --git a/src/scenarios/mauiDesktopBenchmarks/post.py b/src/scenarios/mauiDesktopBenchmarks/post.py new file mode 100644 index 00000000000..07fba6a45f4 --- /dev/null +++ b/src/scenarios/mauiDesktopBenchmarks/post.py @@ -0,0 +1,33 @@ +''' +Post-commands for MAUI Desktop BenchmarkDotNet benchmarks. +Cleans up the cloned maui repo and temporary artifacts. +''' +import os +from performance.common import remove_directory +from performance.logger import setup_loggers, getLogger + +setup_loggers(True) +log = getLogger(__name__) + +# Match the absolute-path resolution used by test.py so cleanup finds the +# checkout regardless of the cwd post.py inherits. +MAUI_REPO_DIRNAME = 'maui_repo' +MAUI_REPO_DIR = os.path.abspath(MAUI_REPO_DIRNAME) + + +def cleanup(): + """Remove the cloned maui repository and any leftover artifacts.""" + if os.path.exists(MAUI_REPO_DIR): + log.info(f'Removing cloned MAUI repo: {MAUI_REPO_DIR}') + remove_directory(MAUI_REPO_DIR) + + # Clean up combined report if still in working directory + combined = 'combined-perf-lab-report.json' + if os.path.exists(combined): + os.remove(combined) + + log.info('Post-commands cleanup complete.') + + +if __name__ == '__main__': + cleanup() diff --git a/src/scenarios/mauiDesktopBenchmarks/pre.py b/src/scenarios/mauiDesktopBenchmarks/pre.py new file mode 100644 index 00000000000..99c7e37499f --- /dev/null +++ b/src/scenarios/mauiDesktopBenchmarks/pre.py @@ -0,0 +1,19 @@ +''' +Pre-commands for MAUI Desktop BenchmarkDotNet benchmarks. +Kept minimal — all heavy lifting (clone, build, patch, run) is in test.py +to keep the correlation payload small. +''' +import argparse +from performance.logger import setup_loggers, getLogger + +setup_loggers(True) +log = getLogger(__name__) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='MAUI Desktop BDN Benchmarks - Pre-commands') + parser.add_argument('-f', '--framework', default='net11.0', + help='Target .NET framework (determines MAUI branch)') + args = parser.parse_args() + log.info(f'MAUI Desktop BDN Benchmarks pre-commands (framework={args.framework})') + log.info('Setup deferred to test.py to minimize correlation payload.') + diff --git a/src/scenarios/mauiDesktopBenchmarks/test.py b/src/scenarios/mauiDesktopBenchmarks/test.py new file mode 100644 index 00000000000..1e42b8f6cef --- /dev/null +++ b/src/scenarios/mauiDesktopBenchmarks/test.py @@ -0,0 +1,296 @@ +''' +MAUI Desktop BenchmarkDotNet benchmarks. + +Handles MAUI-specific setup (clone, branch mapping, dependency build) then +delegates to the shared BDNDesktopHelper for the generic BDN workflow +(patch, build benchmarks, run, collect results). + +Usage: test.py --framework net11.0 --suite all +''' +import os +import re +import shutil +import subprocess +import sys +import urllib.parse +import urllib.request +import zipfile +from argparse import ArgumentParser +from logging import getLogger +from typing import Optional +from performance.common import remove_directory +from performance.logger import setup_loggers +from shared.bdndesktop import BDNDesktopHelper + +# ── MAUI-specific configuration ───────────────────────────────────────────── + +MAUI_REPO_URL = 'https://github.com/dotnet/maui.git' +MAUI_REPO_DIRNAME = 'maui_repo' +# Anchor the clone target to an absolute path so subsequent subprocess +# calls (which may set cwd) keep finding the same checkout. Resolved at +# import time against the current working directory of test.py. +MAUI_REPO_DIR = os.path.abspath(MAUI_REPO_DIRNAME) + +MAUI_BENCHMARK_PROJECTS = { + 'core': 'src/Core/tests/Benchmarks/Core.Benchmarks.csproj', + 'xaml': 'src/Controls/tests/Xaml.Benchmarks/Microsoft.Maui.Controls.Xaml.Benchmarks.csproj', + 'graphics': 'src/Graphics/tests/Graphics.Benchmarks/Graphics.Benchmarks.csproj', +} + +MAUI_SPARSE_CHECKOUT_DIRS = [ + 'src/Core', 'src/Controls', 'src/Graphics', 'src/SingleProject', + 'src/Workload', 'src/Essentials', + 'eng', '.config', +] + +MAUI_BUILD_SOLUTION_FILTER = 'Microsoft.Maui.BuildTasks.slnf' + +# MSBuild properties to disable non-desktop target frameworks. +# MAUI's Directory.Build.props sets these to true unconditionally at multiple +# points; MauiPlatforms is computed from them. In-place replacement is +# required because appending overrides at the end doesn't work (MSBuild +# evaluates top-to-bottom). +DESKTOP_ONLY_PROPS = { + 'IncludeAndroidTargetFrameworks': 'false', + 'IncludeIosTargetFrameworks': 'false', + 'IncludeMacCatalystTargetFrameworks': 'false', + 'IncludeMacOSTargetFrameworks': 'false', + 'IncludeTizenTargetFrameworks': 'false', +} + +# Benchmarks to exclude: these emit millions of log lines per iteration, +# bloating output and slowing runs (BindableProperty readonly errors). +EXCLUDED_BENCHMARKS = [ + '*MauiLoggerWithLoggerMinLevelErrorBenchmarker*', +] + + +_FRAMEWORK_BRANCH_RE = re.compile(r'^(net\d+\.\d+)') + + +def get_branch(framework: str) -> str: + '''Map framework moniker to MAUI repo branch. + + Strips OS/architecture suffixes so values like "net8.0-windows" and + "net11.0-windows10.0.19041.0" map to "net8.0" / "net11.0". + Falls back to "net11.0" for unrecognised input. + ''' + if framework: + m = _FRAMEWORK_BRANCH_RE.match(framework) + if m: + return m.group(1) + return 'net11.0' + + +def _find_git() -> Optional[str]: + '''Find the git executable on PATH or at common Windows locations.''' + git = shutil.which('git') + if git: + return git + + if sys.platform == 'win32': + for candidate in [ + os.path.join(os.environ.get('ProgramFiles', r'C:\Program Files'), 'Git', 'cmd', 'git.exe'), + os.path.join(os.environ.get('ProgramFiles(x86)', r'C:\Program Files (x86)'), 'Git', 'cmd', 'git.exe'), + os.path.join(os.environ.get('ProgramW6432', r'C:\Program Files'), 'Git', 'cmd', 'git.exe'), + ]: + if os.path.isfile(candidate): + return candidate + + return None + + +def _git_sparse_clone(git: str, branch: str, repo_dir: str): + '''Clone using git sparse checkout (preferred — smaller download).''' + log = getLogger() + log.info(f'Using git at: {git}') + + subprocess.run([ + git, 'clone', + '-c', 'core.longpaths=true', + '--depth', '1', + '--filter=blob:none', + '--sparse', + '--branch', branch, + MAUI_REPO_URL, + repo_dir + ], check=True) + + subprocess.run( + [git, 'sparse-checkout', 'set'] + MAUI_SPARSE_CHECKOUT_DIRS, + cwd=repo_dir, check=True) + + +def _zip_download(branch: str, repo_dir: str): + '''Download the repo as a zip archive and extract needed directories. + + Fallback when git is not available (e.g. Helix work items where git is + not on PATH and not installed). + + Uses curl.exe (built into Windows 10+) for the download because Python's + bundled SSL certificates may not include the CA certs trusted by the + machine (common on Helix/corporate environments). + ''' + log = getLogger() + # URL-encode the branch so refs like "release/10.0" round-trip correctly. + archive_url = ( + 'https://github.com/dotnet/maui/archive/refs/heads/' + f'{urllib.parse.quote(branch, safe="")}.zip' + ) + zip_path = 'maui_download.zip' + + log.info(f'git not found — downloading archive from {archive_url}') + + # Use curl.exe (ships with Windows 10+/Server 2016+) which uses the + # Windows certificate store, avoiding Python SSL cert issues on Helix. + curl = shutil.which('curl') or shutil.which('curl.exe') + if curl: + subprocess.run([curl, '-L', '-o', zip_path, '--fail', '-s', '-S', archive_url], check=True) + else: + # Last resort: try urllib with default certs + urllib.request.urlretrieve(archive_url, zip_path) + + log.info(f'Downloaded {os.path.getsize(zip_path) / (1024*1024):.1f} MB') + + os.makedirs(repo_dir, exist_ok=True) + repo_root_real = os.path.realpath(repo_dir) + + # Directories to extract (sparse checkout equivalent + root-level files). + # Also include files directly in parent directories of sparse dirs + # (e.g. src/MultiTargeting.targets, src/PublicAPI.targets) since git + # sparse-checkout includes parent-level files automatically. + sparse_prefixes = [d.rstrip('/') + '/' for d in MAUI_SPARSE_CHECKOUT_DIRS] + parent_dirs = set() + for d in MAUI_SPARSE_CHECKOUT_DIRS: + parts = d.strip('/').split('/') + for i in range(1, len(parts)): + parent_dirs.add('/'.join(parts[:i]) + '/') + parent_dirs = list(parent_dirs) # e.g. ['src/'] + + with zipfile.ZipFile(zip_path) as zf: + # GitHub archives have a top-level dir like "maui-net11.0/" + top_dir = zf.namelist()[0].split('/')[0] + '/' + + for member in zf.namelist(): + if not member.startswith(top_dir): + continue + rel_path = member[len(top_dir):] + if not rel_path: + continue + + # Include: root-level files, sparse directories, and files + # directly in parent directories (not recursing into subdirs) + is_root_file = '/' not in rel_path + in_sparse_dir = any(rel_path.startswith(p) for p in sparse_prefixes) + in_parent_dir = any( + rel_path.startswith(p) and '/' not in rel_path[len(p):] + for p in parent_dirs + ) + + if not is_root_file and not in_sparse_dir and not in_parent_dir: + continue + + target = os.path.join(repo_dir, rel_path) + # Zip-slip guard: refuse to write outside repo_dir even if a + # malicious archive contains entries with ../ or absolute paths. + target_real = os.path.realpath(target) + if not (target_real == repo_root_real or + target_real.startswith(repo_root_real + os.sep)): + log.warning(f'Skipping zip entry outside repo_dir: {member}') + continue + + if member.endswith('/'): + os.makedirs(target, exist_ok=True) + else: + os.makedirs(os.path.dirname(target), exist_ok=True) + with zf.open(member) as src, open(target, 'wb') as dst: + dst.write(src.read()) + + os.remove(zip_path) + log.info('Archive extracted.') + + +def clone_maui_repo(branch: str, repo_dir: str = MAUI_REPO_DIR): + '''Clone or download dotnet/maui at the given branch.''' + log = getLogger() + log.info(f'Acquiring dotnet/maui branch {branch}...') + + if os.path.exists(repo_dir): + remove_directory(repo_dir) + + git = _find_git() + if git: + _git_sparse_clone(git, branch, repo_dir) + else: + _zip_download(branch, repo_dir) + + log.info('MAUI source acquired.') + + +def build_maui_dependencies(repo_dir: str = MAUI_REPO_DIR): + '''Restore dotnet tools and build MAUI's BuildTasks solution filter.''' + log = getLogger() + log.info('Restoring dotnet tools...') + subprocess.run(['dotnet', 'tool', 'restore'], cwd=repo_dir, check=True) + + slnf_path = os.path.join(repo_dir, MAUI_BUILD_SOLUTION_FILTER) + if not os.path.exists(slnf_path): + raise FileNotFoundError( + f'Expected MAUI build solution filter not found: {slnf_path}. ' + f'The dotnet/maui branch layout may have changed; update ' + f'MAUI_BUILD_SOLUTION_FILTER or MAUI_SPARSE_CHECKOUT_DIRS.') + + log.info(f'Building {MAUI_BUILD_SOLUTION_FILTER} (desktop TFMs only)...') + subprocess.run([ + 'dotnet', 'build', + MAUI_BUILD_SOLUTION_FILTER, + '-c', 'Release', + ], cwd=repo_dir, check=True) + + log.info('MAUI dependencies built successfully.') + + +def parse_args(): + parser = ArgumentParser( + description='Run MAUI desktop BDN benchmarks', + epilog='Any unrecognized arguments are forwarded to BenchmarkDotNet ' + '(e.g. --filter MyBenchmark*).') + parser.add_argument('--framework', '-f', default='net11.0', + help='Target .NET framework (determines MAUI repo branch)') + parser.add_argument('--suite', choices=['core', 'xaml', 'graphics', 'all'], + default='all', help='Which benchmark suite to run') + parser.add_argument('--upload-to-perflab-container', action='store_true', + help='Upload results to perflab container') + # Forward unknown args to BenchmarkDotNet. argparse's nargs='*' would + # not capture values that start with '--' (BDN flags), so use + # parse_known_args() instead and pass the remainder through. + return parser.parse_known_args() + + +if __name__ == '__main__': + setup_loggers(True) + args, extra_bdn_args = parse_args() + + # MAUI-specific: clone repo and build dependencies + branch = get_branch(args.framework) + clone_maui_repo(branch) + + # Generic BDN desktop workflow: patch, build benchmarks, run, collect + helper = BDNDesktopHelper( + repo_dir=MAUI_REPO_DIR, + benchmark_projects=MAUI_BENCHMARK_PROJECTS, + disable_props=DESKTOP_ONLY_PROPS, + ) + + # Patch Directory.Build.props BEFORE any builds (including MAUI deps) + helper.patch_directory_build_props() + + # MAUI-specific: build BuildTasks solution filter + build_maui_dependencies() + + # Run the generic BDN workflow. Forward any args we didn't consume + # (e.g. --filter, --maxIterationCount overrides) to BenchmarkDotNet. + bdn_args = list(extra_bdn_args) + if EXCLUDED_BENCHMARKS: + bdn_args.extend(['--exclusion-filter'] + EXCLUDED_BENCHMARKS) + helper.runtests(args.suite, bdn_args, args.upload_to_perflab_container) diff --git a/src/scenarios/shared/bdndesktop.py b/src/scenarios/shared/bdndesktop.py new file mode 100644 index 00000000000..30dec7c4b0d --- /dev/null +++ b/src/scenarios/shared/bdndesktop.py @@ -0,0 +1,445 @@ +''' +Reusable helper for running BenchmarkDotNet benchmarks from external repos +on desktop. + +Handles the generic workflow: + 1. Patch Directory.Build.props to disable unwanted target frameworks + 2. Inject BDN.Extensions (PerfLabExporter) into benchmark projects + 3. Build benchmark projects + 4. Run BDN suites + 5. Collect and upload results + +Callers (e.g. test.py in each scenario) are responsible for repo-specific +setup such as cloning, branch selection, and building repo dependencies. +''' +import os +import re +import sys +import glob +import json +import shutil +import subprocess +import xml.etree.ElementTree as ET +from logging import getLogger +from performance.common import runninginlab + + +# Default BDN run arguments. Tuned to match RecommendedConfig: short +# warmup, capped iterations, 250ms iteration time. Override per-helper via +# the bdn_run_params constructor argument. +DEFAULT_BDN_RUN_PARAMS = [ + '--filter', '*', + '--warmupCount', '1', + '--minIterationCount', '15', + '--maxIterationCount', '20', + '--iterationTime', '250', +] + + +class BDNDesktopHelper(object): + '''Generic helper for running BDN desktop benchmarks from a local repo checkout. + + Args: + repo_dir: Path to the cloned repository root. Stored as + an absolute path so subprocess cwd is stable + regardless of cwd changes by the caller. + benchmark_projects: Dict mapping suite name to csproj relative path + (e.g. {'graphics': 'src/Graphics/.../Graphics.Benchmarks.csproj'}). + disable_props: Optional dict of MSBuild property names to replacement values + to patch in Directory.Build.props (e.g. disable mobile TFMs). + bdn_run_params: Optional list of BDN command-line arguments used + when running each suite. Defaults to + DEFAULT_BDN_RUN_PARAMS. + strict_build: If True, fail the run when *any* listed suite + fails to build. Default False preserves the + tolerant behaviour (skip broken suites, run the + rest) which is convenient while iterating. + ''' + + def __init__(self, repo_dir: str, benchmark_projects: dict, + disable_props: dict = None, + bdn_run_params: list = None, + strict_build: bool = False): + self.repo_dir = os.path.abspath(repo_dir) + self.benchmark_projects = benchmark_projects + self.disable_props = disable_props or {} + self.bdn_run_params = list(bdn_run_params) if bdn_run_params else list(DEFAULT_BDN_RUN_PARAMS) + self.strict_build = strict_build + + # ── Public entry point ────────────────────────────────────────────────── + + def runtests(self, suite: str, bdn_args: list, + upload_to_perflab_container: bool): + ''' + Patch benchmark projects, build, run, and collect BDN results. + + Assumes the caller has already: + - Cloned the repo and built repo-specific dependencies + - Called patch_directory_build_props() if needed (must happen + before any builds, including dependency builds) + ''' + log = getLogger() + + # Patch benchmark csprojs + Program.cs for PerfLabExporter + self.patch_benchmark_projects() + + # Build + built_suites = self.build_benchmark_projects(suite) + + # Run (only suites that built successfully) + if suite == 'all': + suites = [(n, p) for n, p in self.benchmark_projects.items() if n in built_suites] + else: + suites = [(suite, self.benchmark_projects[suite])] + + all_passed = True + for name, csproj_rel in suites: + if not self._run_benchmark(name, csproj_rel, bdn_args): + all_passed = False + + # Collect + self._collect_results(upload_to_perflab_container) + + if not all_passed: + log.error('One or more benchmark suites failed.') + sys.exit(1) + + log.info('All benchmark suites completed.') + + # ── Patch Directory.Build.props ───────────────────────────────────────── + + def patch_directory_build_props(self): + '''Disable unwanted TFMs by replacing property values in-place. + + Handles repos (like MAUI) that set Include*TargetFrameworks=true at + multiple points before computing TargetFrameworks. Appending + overrides at the end doesn't work because MSBuild evaluates + top-to-bottom, so we regex-replace ALL occurrences in-place. + ''' + log = getLogger() + props_path = os.path.join(self.repo_dir, 'Directory.Build.props') + if not os.path.exists(props_path): + log.info('No Directory.Build.props found — skipping TFM patching.') + return + + log.info('Patching Directory.Build.props to disable unwanted TFMs...') + + with open(props_path, 'r', encoding='utf-8-sig') as f: + content = f.read() + + for prop_name, new_value in self.disable_props.items(): + pattern = rf'(<{prop_name}\b[^>]*>)\s*true\s*()' + content, count = re.subn(pattern, rf'\g<1>{new_value}\g<2>', content) + if count > 0: + log.info(f' {prop_name}: replaced {count} occurrence(s)') + else: + log.warning(f' {prop_name}: no occurrences of "true" found to replace') + + with open(props_path, 'w', encoding='utf-8') as f: + f.write(content) + + log.info(' Directory.Build.props patched.') + + # ── BDN.Extensions injection ──────────────────────────────────────────── + + def _find_bdn_extensions(self) -> str: + '''Return the absolute path to BenchmarkDotNet.Extensions.csproj.''' + correlation = os.environ.get('HELIX_CORRELATION_PAYLOAD', '') + if correlation: + candidate = os.path.join( + correlation, 'performance', 'src', 'harness', + 'BenchmarkDotNet.Extensions', 'BenchmarkDotNet.Extensions.csproj') + else: + scenario_dir = os.path.dirname(os.path.abspath(__file__)) + candidate = os.path.normpath(os.path.join( + scenario_dir, '..', '..', 'harness', + 'BenchmarkDotNet.Extensions', 'BenchmarkDotNet.Extensions.csproj')) + + if not os.path.exists(candidate): + raise FileNotFoundError( + f'BenchmarkDotNet.Extensions.csproj not found at {candidate}. ' + f'HELIX_CORRELATION_PAYLOAD={correlation!r}') + + getLogger().info(f'BDN.Extensions located at: {candidate}') + return candidate + + def _inject_bdn_extensions(self, csproj_path: str, bdn_ext_abs: str): + '''Add a ProjectReference to BDN.Extensions and remove existing BDN PackageRef. + + Idempotent: removes any prior ItemGroup with Label="PerfLabInjected" + before adding the new one so re-running against an existing + checkout does not accumulate duplicate ProjectReference entries + (which produce build errors). + ''' + log = getLogger() + log.info(f'Injecting BDN.Extensions reference into {os.path.basename(csproj_path)}') + + csproj_dir = os.path.dirname(os.path.abspath(csproj_path)) + bdn_ext_rel = os.path.relpath(bdn_ext_abs, csproj_dir) + + tree = ET.parse(csproj_path) + root = tree.getroot() + + ns = '' + if root.tag.startswith('{'): + ns = root.tag.split('}')[0] + '}' + + # Remove any prior PerfLabInjected ItemGroup so re-runs don't + # accumulate duplicate ProjectReferences. + existing_injected = [ig for ig in root.findall(f'{ns}ItemGroup') + if ig.get('Label') == 'PerfLabInjected'] + for ig in existing_injected: + root.remove(ig) + if existing_injected: + log.info(f' Removed {len(existing_injected)} prior PerfLabInjected ItemGroup(s)') + + # Remove BDN package references that conflict with the injected + # ProjectReference to BenchmarkDotNet.Extensions (which itself + # references BenchmarkDotNet + BenchmarkDotNet.Annotations as + # ProjectReferences). Leaving them as PackageReferences in the + # benchmark csproj would pull in a different (likely older) BDN + # version and produce duplicate-type errors at build time. + # Optional sub-packages (e.g. BenchmarkDotNet.Diagnostics.*) are + # left in place because BDN.Extensions does not reference them. + bdn_packages_to_remove = {'BenchmarkDotNet', 'BenchmarkDotNet.Annotations'} + for item_group in root.findall(f'{ns}ItemGroup'): + for pkg_ref in item_group.findall(f'{ns}PackageReference'): + include = pkg_ref.get('Include', '') + if include in bdn_packages_to_remove: + item_group.remove(pkg_ref) + log.info(f' Removed PackageReference: {include}') + + # Add ProjectReference to BDN.Extensions + item_group = ET.SubElement(root, f'{ns}ItemGroup') + item_group.set('Label', 'PerfLabInjected') + proj_ref = ET.SubElement(item_group, f'{ns}ProjectReference') + proj_ref.set('Include', bdn_ext_rel) + + tree.write(csproj_path, xml_declaration=True, encoding='utf-8') + log.info(f' Added ProjectReference: {bdn_ext_rel}') + + def _patch_program_cs(self, program_cs_path: str): + '''Patch Program.cs to add PerfLabExporter via ManualConfig.''' + log = getLogger() + log.info(f'Patching {os.path.basename(program_cs_path)} for PerfLabExporter') + + with open(program_cs_path, 'r', encoding='utf-8-sig') as f: + content = f.read() + + usings_to_add = [ + 'using BenchmarkDotNet.Configs;', + 'using BenchmarkDotNet.Extensions;', + 'using System;', + 'using System.Collections.Generic;', + 'using System.IO;', + 'using System.Linq;', + ] + insert_block = '' + for u in usings_to_add: + if u not in content: + insert_block += u + '\n' + if insert_block: + content = insert_block + content + + # ManualConfig without MandatoryCategoryValidator (external benchmarks + # may not use [BenchmarkCategory]) + new_run_call = ( + 'var argsList = args.ToList();\n' + ' argsList = CommandLineOptions.ParseAndRemoveStringsParameter(\n' + ' argsList, "--exclusion-filter", out var exclusionFilterValue);\n' + ' var config = ManualConfig.Create(DefaultConfig.Instance)\n' + ' .WithArtifactsPath(Path.Combine(\n' + ' Path.GetDirectoryName(typeof(Program).Assembly.Location),\n' + ' "BenchmarkDotNet.Artifacts"))\n' + ' .AddFilter(new ExclusionFilter(exclusionFilterValue));\n' + ' if (Environment.GetEnvironmentVariable("PERFLAB_INLAB") == "1")\n' + ' config = config.AddExporter(new PerfLabExporter());\n' + ' BenchmarkSwitcher\n' + ' .FromAssembly(typeof(Program).Assembly)\n' + ' .Run(argsList.ToArray(), config);' + ) + + patterns = [ + 'BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);', + 'BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args)', + 'BenchmarkSwitcher.FromAssembly (typeof (Program).Assembly).Run (args);', + 'BenchmarkSwitcher.FromAssembly (typeof (Program).Assembly).Run (args)', + ] + + replaced = False + for pattern in patterns: + if pattern in content: + content = content.replace(pattern, new_run_call) + replaced = True + break + + if not replaced: + raise RuntimeError( + f'Could not find a BenchmarkSwitcher.Run pattern in ' + f'{program_cs_path}. PerfLabExporter would not be wired up ' + f'and the run would silently produce no perf-lab-report.json. ' + f'Update _patch_program_cs patterns to match this file.') + + with open(program_cs_path, 'w', encoding='utf-8') as f: + f.write(content) + + log.info(' Patched successfully.') + + def patch_benchmark_projects(self): + '''Inject BDN.Extensions and PerfLabExporter into all benchmark projects.''' + bdn_ext_abs = self._find_bdn_extensions() + + for name, csproj_rel in self.benchmark_projects.items(): + csproj_path = os.path.join(self.repo_dir, csproj_rel) + project_dir = os.path.dirname(csproj_path) + program_cs = os.path.join(project_dir, 'Program.cs') + + if not os.path.exists(csproj_path): + getLogger().warning(f'Benchmark project not found: {csproj_path}') + continue + + self._inject_bdn_extensions(csproj_path, bdn_ext_abs) + + if os.path.exists(program_cs): + self._patch_program_cs(program_cs) + else: + getLogger().warning(f'Program.cs not found for {name}') + + # ── Build benchmarks ──────────────────────────────────────────────────── + + def build_benchmark_projects(self, suite: str) -> set: + '''Build benchmark projects. Returns the set of suite names that built successfully.''' + log = getLogger() + if suite == 'all': + projects = list(self.benchmark_projects.items()) + else: + projects = [(suite, self.benchmark_projects[suite])] + + built = set() + for name, csproj_rel in projects: + csproj_path = os.path.join(self.repo_dir, csproj_rel) + if not os.path.exists(csproj_path): + log.warning(f'Benchmark project not found, skipping: {csproj_path}') + continue + + log.info(f'Building benchmark: {name}') + result = subprocess.run([ + 'dotnet', 'build', + csproj_rel, + '-c', 'Release', + ], cwd=self.repo_dir) + + if result.returncode == 0: + built.add(name) + else: + log.warning(f'Build failed for {name} (exit code {result.returncode}) — skipping this suite') + + if built: + log.info(f'Successfully built: {", ".join(sorted(built))}') + else: + log.error('No benchmark projects built successfully.') + sys.exit(1) + + failed = set(name for name, _ in projects) - built + if failed: + msg = f'The following suites failed to build: {", ".join(sorted(failed))}' + if self.strict_build: + log.error(msg) + sys.exit(1) + log.warning(f'{msg} — skipping (strict_build=False)') + + return built + + # ── Run benchmarks ────────────────────────────────────────────────────── + + def _run_benchmark(self, name: str, csproj_rel: str, extra_bdn_args: list) -> bool: + log = getLogger() + csproj_path = os.path.join(self.repo_dir, csproj_rel) + if not os.path.exists(csproj_path): + log.warning(f'Benchmark project not found: {csproj_path}') + return False + + log.info(f'Running benchmark suite: {name}') + + cmd = [ + 'dotnet', 'run', + '-c', 'Release', + '--no-build', + '--project', csproj_rel, + '--', + ] + list(self.bdn_run_params) + extra_bdn_args + + result = subprocess.run(cmd, cwd=self.repo_dir) + if result.returncode != 0: + log.error(f'Benchmark suite {name} failed with exit code {result.returncode}') + return False + + log.info(f'Benchmark suite {name} completed successfully.') + return True + + # ── Result collection ─────────────────────────────────────────────────── + + def _collect_results(self, upload_to_perflab_container: bool): + '''Collect perf-lab-report.json files from BDN artifacts.''' + log = getLogger() + upload_root = os.environ.get('HELIX_WORKITEM_UPLOAD_ROOT', '') + + report_pattern = os.path.join(self.repo_dir, '**', '*perf-lab-report.json') + report_files = glob.glob(report_pattern, recursive=True) + + if not report_files: + log.warning('No *perf-lab-report.json files found. ' + 'PerfLabExporter may not have been active (PERFLAB_INLAB not set?).') + return + + log.info(f'Found {len(report_files)} perf-lab-report.json file(s)') + + # Combine all reports into a single file + combined = [] + for report_file in report_files: + log.info(f' Collecting: {report_file}') + try: + with open(report_file, 'r', encoding='utf-8') as f: + data = json.load(f) + if isinstance(data, list): + combined.extend(data) + else: + combined.append(data) + except (json.JSONDecodeError, IOError) as e: + log.warning(f' Failed to read {report_file}: {e}') + + if combined: + combined_path = 'combined-perf-lab-report.json' + with open(combined_path, 'w', encoding='utf-8') as f: + json.dump(combined, f, indent=2) + log.info(f'Combined report: {combined_path} ({len(combined)} result(s))') + + if upload_root: + dest = os.path.join(upload_root, combined_path) + shutil.copy2(combined_path, dest) + log.info(f'Copied combined report to {dest}') + + # Copy each per-suite report preserving the path under + # repo_dir to avoid filename collisions when multiple + # suites emit reports with the same basename. + for report_file in report_files: + rel = os.path.relpath(report_file, self.repo_dir) + # Flatten path separators so the upload is one level + # deep but still uniquely named per source location. + flat = rel.replace(os.sep, '_').replace('/', '_') + dest = os.path.join(upload_root, flat) + shutil.copy2(report_file, dest) + log.info(f' Copied {rel} -> {flat}') + + # Also upload via perflab container if requested + if upload_to_perflab_container and runninginlab(): + try: + from performance.constants import UPLOAD_CONTAINER, UPLOAD_STORAGE_URI, UPLOAD_QUEUE + import upload + globpath = os.path.join(self.repo_dir, '**', '*perf-lab-report.json') + upload_code = upload.upload(globpath, UPLOAD_CONTAINER, UPLOAD_QUEUE, UPLOAD_STORAGE_URI) + log.info(f'BDN Desktop Benchmarks Upload Code: {upload_code}') + if upload_code != 0: + sys.exit(upload_code) + except ImportError: + log.warning('Upload module not available — skipping perflab container upload') From c6d9388bbc3e1c00da7a9897136035b49dbc319b Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 11 May 2026 13:55:33 -0700 Subject: [PATCH 2/3] Use shared performance.common helpers instead of rolling our own MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reuse improvements across bdndesktop.py and test.py to align with the rest of the scenarios codebase: - Replace direct `subprocess.run` calls with `performance.common.RunCommand`. RunCommand provides consistent `$ cmdline` logging, working-directory handling, and raises CalledProcessError on non-zero exit, matching how every other scenario (precommands, runner, androidhelper, bdnandroid) invokes `dotnet` / `git` / `curl`. For places where we deliberately tolerate failure (per-suite `dotnet build` and `dotnet run`), catch CalledProcessError and surface the returncode in the warning. - Replace direct env-var reads with the shared accessors: - `os.environ.get('HELIX_CORRELATION_PAYLOAD', '')` → `helixpayload()` - `os.environ.get('HELIX_WORKITEM_UPLOAD_ROOT', '')` → `helixuploadroot()` - Replace the manual `__file__` + `../../harness/...` navigation in `_find_bdn_extensions` with `get_repo_root_path() / 'src' / 'harness' / ...`. Same destination, but works regardless of where `bdndesktop.py` lives in the tree. - Replace `sys.platform == 'win32'` in `_find_git` with `performance.common.iswin()` to match repo style. `upload.upload` was already being reused for the perflab container upload path; this commit removes the remaining ad-hoc subprocess / env-var usage so the new scenario stops duplicating functionality that already exists in the repo. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/mauiDesktopBenchmarks/test.py | 28 ++++++------ src/scenarios/shared/bdndesktop.py | 47 ++++++++++++--------- 2 files changed, 40 insertions(+), 35 deletions(-) diff --git a/src/scenarios/mauiDesktopBenchmarks/test.py b/src/scenarios/mauiDesktopBenchmarks/test.py index 1e42b8f6cef..ed9a43451f5 100644 --- a/src/scenarios/mauiDesktopBenchmarks/test.py +++ b/src/scenarios/mauiDesktopBenchmarks/test.py @@ -10,15 +10,13 @@ import os import re import shutil -import subprocess -import sys import urllib.parse import urllib.request import zipfile from argparse import ArgumentParser from logging import getLogger from typing import Optional -from performance.common import remove_directory +from performance.common import RunCommand, iswin, remove_directory from performance.logger import setup_loggers from shared.bdndesktop import BDNDesktopHelper @@ -88,7 +86,7 @@ def _find_git() -> Optional[str]: if git: return git - if sys.platform == 'win32': + if iswin(): for candidate in [ os.path.join(os.environ.get('ProgramFiles', r'C:\Program Files'), 'Git', 'cmd', 'git.exe'), os.path.join(os.environ.get('ProgramFiles(x86)', r'C:\Program Files (x86)'), 'Git', 'cmd', 'git.exe'), @@ -102,10 +100,9 @@ def _find_git() -> Optional[str]: def _git_sparse_clone(git: str, branch: str, repo_dir: str): '''Clone using git sparse checkout (preferred — smaller download).''' - log = getLogger() - log.info(f'Using git at: {git}') + getLogger().info(f'Using git at: {git}') - subprocess.run([ + RunCommand([ git, 'clone', '-c', 'core.longpaths=true', '--depth', '1', @@ -113,12 +110,12 @@ def _git_sparse_clone(git: str, branch: str, repo_dir: str): '--sparse', '--branch', branch, MAUI_REPO_URL, - repo_dir - ], check=True) + repo_dir, + ], verbose=True).run() - subprocess.run( + RunCommand( [git, 'sparse-checkout', 'set'] + MAUI_SPARSE_CHECKOUT_DIRS, - cwd=repo_dir, check=True) + verbose=True).run(repo_dir) def _zip_download(branch: str, repo_dir: str): @@ -145,7 +142,8 @@ def _zip_download(branch: str, repo_dir: str): # Windows certificate store, avoiding Python SSL cert issues on Helix. curl = shutil.which('curl') or shutil.which('curl.exe') if curl: - subprocess.run([curl, '-L', '-o', zip_path, '--fail', '-s', '-S', archive_url], check=True) + RunCommand([curl, '-L', '-o', zip_path, '--fail', '-s', '-S', archive_url], + verbose=True).run() else: # Last resort: try urllib with default certs urllib.request.urlretrieve(archive_url, zip_path) @@ -231,7 +229,7 @@ def build_maui_dependencies(repo_dir: str = MAUI_REPO_DIR): '''Restore dotnet tools and build MAUI's BuildTasks solution filter.''' log = getLogger() log.info('Restoring dotnet tools...') - subprocess.run(['dotnet', 'tool', 'restore'], cwd=repo_dir, check=True) + RunCommand(['dotnet', 'tool', 'restore'], verbose=True).run(repo_dir) slnf_path = os.path.join(repo_dir, MAUI_BUILD_SOLUTION_FILTER) if not os.path.exists(slnf_path): @@ -241,11 +239,11 @@ def build_maui_dependencies(repo_dir: str = MAUI_REPO_DIR): f'MAUI_BUILD_SOLUTION_FILTER or MAUI_SPARSE_CHECKOUT_DIRS.') log.info(f'Building {MAUI_BUILD_SOLUTION_FILTER} (desktop TFMs only)...') - subprocess.run([ + RunCommand([ 'dotnet', 'build', MAUI_BUILD_SOLUTION_FILTER, '-c', 'Release', - ], cwd=repo_dir, check=True) + ], verbose=True).run(repo_dir) log.info('MAUI dependencies built successfully.') diff --git a/src/scenarios/shared/bdndesktop.py b/src/scenarios/shared/bdndesktop.py index 30dec7c4b0d..06f83022b46 100644 --- a/src/scenarios/shared/bdndesktop.py +++ b/src/scenarios/shared/bdndesktop.py @@ -18,10 +18,16 @@ import glob import json import shutil -import subprocess import xml.etree.ElementTree as ET from logging import getLogger -from performance.common import runninginlab +from subprocess import CalledProcessError +from performance.common import ( + RunCommand, + get_repo_root_path, + helixpayload, + helixuploadroot, + runninginlab, +) # Default BDN run arguments. Tuned to match RecommendedConfig: short @@ -144,16 +150,17 @@ def patch_directory_build_props(self): def _find_bdn_extensions(self) -> str: '''Return the absolute path to BenchmarkDotNet.Extensions.csproj.''' - correlation = os.environ.get('HELIX_CORRELATION_PAYLOAD', '') + correlation = helixpayload() if correlation: candidate = os.path.join( correlation, 'performance', 'src', 'harness', 'BenchmarkDotNet.Extensions', 'BenchmarkDotNet.Extensions.csproj') else: - scenario_dir = os.path.dirname(os.path.abspath(__file__)) - candidate = os.path.normpath(os.path.join( - scenario_dir, '..', '..', 'harness', - 'BenchmarkDotNet.Extensions', 'BenchmarkDotNet.Extensions.csproj')) + # Resolve against the performance repo root via the shared + # helper so this works no matter where bdndesktop.py lives. + candidate = os.path.join( + get_repo_root_path(), 'src', 'harness', + 'BenchmarkDotNet.Extensions', 'BenchmarkDotNet.Extensions.csproj') if not os.path.exists(candidate): raise FileNotFoundError( @@ -323,16 +330,15 @@ def build_benchmark_projects(self, suite: str) -> set: continue log.info(f'Building benchmark: {name}') - result = subprocess.run([ - 'dotnet', 'build', - csproj_rel, - '-c', 'Release', - ], cwd=self.repo_dir) - - if result.returncode == 0: + try: + RunCommand([ + 'dotnet', 'build', + csproj_rel, + '-c', 'Release', + ], verbose=True).run(self.repo_dir) built.add(name) - else: - log.warning(f'Build failed for {name} (exit code {result.returncode}) — skipping this suite') + except CalledProcessError as e: + log.warning(f'Build failed for {name} (exit code {e.returncode}) — skipping this suite') if built: log.info(f'Successfully built: {", ".join(sorted(built))}') @@ -369,9 +375,10 @@ def _run_benchmark(self, name: str, csproj_rel: str, extra_bdn_args: list) -> bo '--', ] + list(self.bdn_run_params) + extra_bdn_args - result = subprocess.run(cmd, cwd=self.repo_dir) - if result.returncode != 0: - log.error(f'Benchmark suite {name} failed with exit code {result.returncode}') + try: + RunCommand(cmd, verbose=True).run(self.repo_dir) + except CalledProcessError as e: + log.error(f'Benchmark suite {name} failed with exit code {e.returncode}') return False log.info(f'Benchmark suite {name} completed successfully.') @@ -382,7 +389,7 @@ def _run_benchmark(self, name: str, csproj_rel: str, extra_bdn_args: list) -> bo def _collect_results(self, upload_to_perflab_container: bool): '''Collect perf-lab-report.json files from BDN artifacts.''' log = getLogger() - upload_root = os.environ.get('HELIX_WORKITEM_UPLOAD_ROOT', '') + upload_root = helixuploadroot() or '' report_pattern = os.path.join(self.repo_dir, '**', '*perf-lab-report.json') report_files = glob.glob(report_pattern, recursive=True) From 5ee5f3bc5977e6f39bcfcb37123fc6a9b639e93f Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 11 May 2026 14:04:54 -0700 Subject: [PATCH 3/3] Align _collect_results with the benchmarks_ci.py upload pattern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/benchmarks_ci.py already implements this same pattern (lines ~286-328): glob *perf-lab-report.json files, copy each by basename to HELIX_WORKITEM_UPLOAD_ROOT, and write a combined-perf-lab-report.json directly into the upload root. bdndesktop._collect_results was rolling its own variant — writing the combined file to cwd first, then copying it, and flattening per-suite report paths to avoid hypothetical basename collisions. Changes: - Write combined-perf-lab-report.json directly into the upload root (no local intermediate, no extra shutil.copy2). - Copy each per-suite report by basename. BDN already namespaces artifacts by `.` so collisions don't happen in practice across our Core / Xaml / Graphics suites; keep a defensive `Basename collision` warning if it ever does. - Drop the now-unneeded cleanup of combined-perf-lab-report.json from post.py since we never create the local file anymore. The shared pattern isn't extracted into a function yet (it would require touching scripts/benchmarks_ci.py which is out of scope), but keeping the two implementations in sync makes a future extraction trivial. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/mauiDesktopBenchmarks/post.py | 7 +- src/scenarios/shared/bdndesktop.py | 80 +++++++++++---------- 2 files changed, 42 insertions(+), 45 deletions(-) diff --git a/src/scenarios/mauiDesktopBenchmarks/post.py b/src/scenarios/mauiDesktopBenchmarks/post.py index 07fba6a45f4..b06131b5bdd 100644 --- a/src/scenarios/mauiDesktopBenchmarks/post.py +++ b/src/scenarios/mauiDesktopBenchmarks/post.py @@ -16,16 +16,11 @@ def cleanup(): - """Remove the cloned maui repository and any leftover artifacts.""" + """Remove the cloned maui repository.""" if os.path.exists(MAUI_REPO_DIR): log.info(f'Removing cloned MAUI repo: {MAUI_REPO_DIR}') remove_directory(MAUI_REPO_DIR) - # Clean up combined report if still in working directory - combined = 'combined-perf-lab-report.json' - if os.path.exists(combined): - os.remove(combined) - log.info('Post-commands cleanup complete.') diff --git a/src/scenarios/shared/bdndesktop.py b/src/scenarios/shared/bdndesktop.py index 06f83022b46..7cdf9480528 100644 --- a/src/scenarios/shared/bdndesktop.py +++ b/src/scenarios/shared/bdndesktop.py @@ -387,12 +387,18 @@ def _run_benchmark(self, name: str, csproj_rel: str, extra_bdn_args: list) -> bo # ── Result collection ─────────────────────────────────────────────────── def _collect_results(self, upload_to_perflab_container: bool): - '''Collect perf-lab-report.json files from BDN artifacts.''' + '''Collect perf-lab-report.json files from BDN artifacts. + + Mirrors the canonical pattern in scripts/benchmarks_ci.py: copy each + per-suite report to HELIX_WORKITEM_UPLOAD_ROOT under its basename + (BDN already namespaces reports by project name) and write a + combined-perf-lab-report.json directly into the upload root. + ''' log = getLogger() - upload_root = helixuploadroot() or '' + upload_root = helixuploadroot() - report_pattern = os.path.join(self.repo_dir, '**', '*perf-lab-report.json') - report_files = glob.glob(report_pattern, recursive=True) + reports_globpath = os.path.join(self.repo_dir, '**', '*perf-lab-report.json') + report_files = glob.glob(reports_globpath, recursive=True) if not report_files: log.warning('No *perf-lab-report.json files found. ' @@ -401,42 +407,38 @@ def _collect_results(self, upload_to_perflab_container: bool): log.info(f'Found {len(report_files)} perf-lab-report.json file(s)') - # Combine all reports into a single file - combined = [] - for report_file in report_files: - log.info(f' Collecting: {report_file}') - try: - with open(report_file, 'r', encoding='utf-8') as f: - data = json.load(f) - if isinstance(data, list): - combined.extend(data) - else: - combined.append(data) - except (json.JSONDecodeError, IOError) as e: - log.warning(f' Failed to read {report_file}: {e}') - - if combined: - combined_path = 'combined-perf-lab-report.json' - with open(combined_path, 'w', encoding='utf-8') as f: - json.dump(combined, f, indent=2) - log.info(f'Combined report: {combined_path} ({len(combined)} result(s))') - - if upload_root: - dest = os.path.join(upload_root, combined_path) - shutil.copy2(combined_path, dest) - log.info(f'Copied combined report to {dest}') - - # Copy each per-suite report preserving the path under - # repo_dir to avoid filename collisions when multiple - # suites emit reports with the same basename. + if upload_root is not None: + # Copy individual reports. BDN names each artifact after the + # project + benchmark type, so basenames are unique across our + # suites in practice; warn if that ever stops being true. + seen = set() + for report_file in report_files: + basename = os.path.basename(report_file) + if basename in seen: + log.warning(f' Basename collision in upload root, overwriting: {basename}') + seen.add(basename) + shutil.copy(report_file, os.path.join(upload_root, basename)) + log.info(f' Copied {basename} to upload root') + + # Write the combined report directly to the upload root (no + # local intermediate file to clean up afterwards). + combined_path = os.path.join(upload_root, 'combined-perf-lab-report.json') + with open(combined_path, 'w', encoding='utf-8') as out: + combined = [] for report_file in report_files: - rel = os.path.relpath(report_file, self.repo_dir) - # Flatten path separators so the upload is one level - # deep but still uniquely named per source location. - flat = rel.replace(os.sep, '_').replace('/', '_') - dest = os.path.join(upload_root, flat) - shutil.copy2(report_file, dest) - log.info(f' Copied {rel} -> {flat}') + try: + with open(report_file, 'r', encoding='utf-8') as f: + data = json.load(f) + if isinstance(data, list): + combined.extend(data) + else: + combined.append(data) + except (json.JSONDecodeError, IOError) as e: + log.warning(f' Failed to read {report_file}: {e}') + json.dump(combined, out) + log.info(f'Combined report: {combined_path} ({len(combined)} result(s))') + else: + log.info('HELIX_WORKITEM_UPLOAD_ROOT not set — skipping upload-root copy.') # Also upload via perflab container if requested if upload_to_perflab_container and runninginlab():