diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj index 1db0b34c6ba..0346b33731f 100644 --- a/eng/performance/maui_scenarios_ios.proj +++ b/eng/performance/maui_scenarios_ios.proj @@ -2,45 +2,50 @@ - true + <_IsInnerLoop Condition="'$(RunKind)' == 'maui_scenarios_ios_innerloop'">true + true - + $(Python) post.py scenarios_out $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\ $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/ - + <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'mono'">/p:UseMonoRuntime=true <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr'">/p:UseMonoRuntime=false - - - - <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr'">$(_MSBuildArgs);/p:PublishReadyToRunStripDebugInfo=false;/p:PublishReadyToRunStripInliningInfo=false - - <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr' and '$(CodegenType)' == 'Interpreter'">$(_MSBuildArgs);/p:PublishReadyToRun=false - <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr' and '$(CodegenType)' == 'NativeAOT'">$(_MSBuildArgs);/p:PublishAot=true;/p:PublishAotUsingRuntimePack=true + + + <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'mono'">/p:UseMonoRuntime=true + <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr'">/p:UseMonoRuntime=false + iossimulator-x64 + <_MSBuildArgs>$(_MSBuildArgs) /p:RuntimeIdentifier=$(iOSRid) + <_MSBuildArgs Condition="'$(iOSRid)' == 'ios-arm64'">$(_MSBuildArgs) /p:EnableCodeSigning=false + <_MSBuildArgs>$(_MSBuildArgs) /p:MtouchLink=None + $(RuntimeFlavor)_Default + 3 + <_MacEnvVars>export DOTNET_ROOT=$HELIX_CORRELATION_PAYLOAD/dotnet;export DOTNET_CLI_TELEMETRY_OPTOUT=1;export DOTNET_MULTILEVEL_LOOKUP=0;export NUGET_PACKAGES=$HELIX_WORKITEM_ROOT/.packages;export PATH=$HELIX_CORRELATION_PAYLOAD/dotnet:$PATH - - - + + - 00:30 + 00:30 + 02:30 - + netios $(ScenariosDir)%(ScenarioDirectoryName) @@ -67,7 +72,7 @@ - + XCODE_PATH=`find /Applications -maxdepth 1 -type d -name 'Xcode_*.app' | sort -t_ -k2 -V | tail -1` && echo "Selected Xcode: $XCODE_PATH" && sudo xcode-select -s "$XCODE_PATH" && $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c $(BuildConfig) -r ios-arm64 --msbuild="$(_MSBuildArgs)" --binlog $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)/%(PreparePayloadWorkItem.ScenarioDirectoryName).binlog -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName) && rm -rf app/obj app/bin && cd ../ && zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName) @@ -75,7 +80,7 @@ - + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName) $HELIX_WORKITEM_ROOT/pub $(Python) test.py sod --scenario-name "%(Identity)" $(ScenarioArgs) @@ -124,7 +129,7 @@ - - + + + mauiiosinnerloop + $(ScenariosDir)%(ScenarioDirectoryName) + + + + + + XCODE_PATH=`find /Applications -maxdepth 1 -type d -name 'Xcode_*.app' | sort -t_ -k2 -V | tail -1` && echo "Selected Xcode: $XCODE_PATH" && sudo xcode-select -s "$XCODE_PATH" && $(Python) pre.py default -f $(PERFLAB_Framework)-ios + %(PreparePayloadWorkItem.PayloadDirectory) + + + + + + $(_MacEnvVars);export IOS_RID=$(iOSRid);$(Python) setup_helix.py $(PERFLAB_Framework)-ios "$(_MSBuildArgs)" || exit $? + $(Python) test.py iosinnerloop --csproj-path app/MauiiOSInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --bundle-id com.companyname.mauiiosinnerloop -f $(PERFLAB_Framework)-ios -c Debug --msbuild-args "$(_MSBuildArgs)" --device-type simulator --inner-loop-iterations $(InnerLoopIterations) --scenario-name "%(Identity)" $(ScenarioArgs) + export IOS_RID=$(iOSRid);$(Python) post.py + output.log + + + + + + $(_MacEnvVars);export IOS_RID=$(iOSRid);$(Python) setup_helix.py $(PERFLAB_Framework)-ios "$(_MSBuildArgs)" || exit $? + $(Python) test.py iosinnerloop --csproj-path app/MauiiOSInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --bundle-id com.companyname.mauiiosinnerloop -f $(PERFLAB_Framework)-ios -c Debug --msbuild-args "$(_MSBuildArgs)" --device-type device --inner-loop-iterations $(InnerLoopIterations) --scenario-name "%(Identity)" $(ScenarioArgs) + export IOS_RID=$(iOSRid);$(Python) post.py + output.log + + - - + export PYTHONPATH=$ORIGPYPATH;$(HelixPostCommands) diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index 4c3a62d1a72..d296a326333 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -565,6 +565,86 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} + # Maui iOS Inner Loop (Mono) - Debug - Simulator + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios_innerloop + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_InnerLoop_Simulator + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui iOS Inner Loop (CoreCLR) - Debug - Simulator + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios_innerloop + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: CoreCLR_InnerLoop_Simulator + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui iOS Inner Loop (Mono) - Debug - Device + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios_innerloop + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_InnerLoop_Device + runEnvVars: + - iOSRid=ios-arm64 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui iOS Inner Loop (CoreCLR) - Debug - Device + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios_innerloop + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: CoreCLR_InnerLoop_Device + runEnvVars: + - iOSRid=ios-arm64 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + # Maui scenario benchmarks - ${{ if false }}: - template: /eng/pipelines/templates/build-machine-matrix.yml diff --git a/eng/pipelines/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml index 4fe69d8c312..68b3b16df4f 100644 --- a/eng/pipelines/templates/run-performance-job.yml +++ b/eng/pipelines/templates/run-performance-job.yml @@ -190,7 +190,7 @@ jobs: - '--is-scenario' - ${{ if ne(length(parameters.runEnvVars), 0) }}: - "--run-env-vars ${{ join(' ', parameters.runEnvVars)}}" - - ${{ if and(in(parameters.runKind, 'maui_scenarios_ios', 'maui_scenarios_android'), ne(parameters.runtimeFlavor, '')) }}: + - ${{ if and(in(parameters.runKind, 'maui_scenarios_ios', 'maui_scenarios_android', 'maui_scenarios_ios_innerloop'), ne(parameters.runtimeFlavor, '')) }}: - '--runtime-flavor ${{ parameters.runtimeFlavor }}' - ${{ if ne(parameters.osVersion, '') }}: - '--os-version ${{ parameters.osVersion }}' diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index 47dee990e8b..f89e7253d5f 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -590,6 +590,15 @@ def get_run_configurations( if build_config is not None and build_config != DEFAULT_BUILD_CONFIG: configurations["BuildConfig"] = build_config + # .NET MAUI iOS inner loop (build+deploy) scenarios + if run_kind == "maui_scenarios_ios_innerloop": + if not runtime_flavor in ("mono", "coreclr"): + raise Exception("Runtime flavor must be specified for maui_scenarios_ios_innerloop") + configurations["CodegenType"] = str(codegen_type) + configurations["RuntimeType"] = str(runtime_flavor) + if build_config is not None and build_config != DEFAULT_BUILD_CONFIG: + configurations["BuildConfig"] = build_config + return configurations def get_work_item_command(os_group: str, target_csproj: str, architecture: str, perf_lab_framework: str, internal: bool, wasm: bool, bdn_artifacts_dir: str, wasm_coreclr: bool = False, only_sanity_check: bool = False): @@ -1164,6 +1173,12 @@ def publish_dotnet_app_to_payload(payload_dir_name: str, csproj_path: str, self_ os.environ["CodegenType"] = args.codegen_type or '' os.environ["BuildConfig"] = args.build_config or DEFAULT_BUILD_CONFIG + # Propagate run_env_vars to os.environ so they reach MSBuild + # evaluation as properties (e.g., iOSRid for device builds). + if args.run_env_vars: + for key, value in args.run_env_vars.items(): + os.environ[key] = value + # TODO: See if these commands are needed for linux as they were being called before but were failing. if args.os_group == "windows" or args.os_group == "osx": break_system_packages = ["--break-system-packages"] if args.os_group == "osx" else [] @@ -1193,7 +1208,7 @@ def publish_dotnet_app_to_payload(payload_dir_name: str, csproj_path: str, self_ verbose=True).run() # Search for additional binlogs generated by the maui scenarios prepare payload work items to copy to the artifacts log dir - if args.run_kind in ["maui_scenarios_android", "maui_scenarios_ios"]: + if args.run_kind in ["maui_scenarios_android", "maui_scenarios_ios", "maui_scenarios_ios_innerloop"]: for binlog_path in glob(os.path.join(payload_dir, "scenarios_out", "**", "*.binlog"), recursive=True): shutil.copy(binlog_path, ci_artifacts_log_dir) @@ -1324,10 +1339,20 @@ def get_work_item_command_for_artifact_dir(artifact_dir: str): only_sanity_check=args.only_sanity_check, ios_strip_symbols=args.ios_strip_symbols, ios_llvm_build=args.ios_llvm_build, + ios_rid=args.run_env_vars.get("iOSRid"), fail_on_test_failure=fail_on_test_failure, scenario_arguments=scenario_arguments or None) if args.send_to_helix: + # Re-apply run_env_vars so they reach SendToHelix MSBuild evaluation. + # The env was snapshot/restored earlier (environ_copy), and while + # os.environ.update() preserves new keys, some shell wrappers may not + # inherit them reliably. Explicitly re-setting ensures properties like + # iOSRid (used in .proj ItemGroup conditions) are available. + if args.run_env_vars: + for key, value in args.run_env_vars.items(): + os.environ[key] = value + perf_send_to_helix(perf_send_to_helix_args) results_glob = os.path.join(helix_results_destination_dir, '**', '*perf-lab-report.json') diff --git a/scripts/send_to_helix.py b/scripts/send_to_helix.py index f3266623e3a..d88ff0b0986 100644 --- a/scripts/send_to_helix.py +++ b/scripts/send_to_helix.py @@ -71,6 +71,7 @@ class PerfSendToHelixArgs: linking_type: Optional[str] = None python: Optional[str] = None affinity: Optional[str] = None + ios_rid: Optional[str] = None ios_strip_symbols: Optional[bool] = None ios_llvm_build: Optional[bool] = None scenario_arguments: Optional[list[str]] = None @@ -111,6 +112,7 @@ def set_env_var(name: str, value: Union[str, bool, list[str], timedelta, int, No set_env_var("RuntimeFlavor", self.runtime_flavor) set_env_var("CodegenType", self.codegen_type) set_env_var("LinkingType", self.linking_type) + set_env_var("iOSRid", self.ios_rid) set_env_var("iOSStripSymbols", self.ios_strip_symbols) set_env_var("iOSLlvmBuild", self.ios_llvm_build) set_env_var("TargetCsproj", self.target_csproj) @@ -142,5 +144,11 @@ def perf_send_to_helix(args: PerfSendToHelixArgs): binlog_dest = os.path.join(args.performance_repo_dir, "artifacts", "log", args.build_config, "SendToHelix.binlog") send_params = [args.project_file, "/restore", "/t:Test", f"/bl:{binlog_dest}"] + # Pass iOSRid explicitly as an MSBuild property so it reaches .proj + # evaluation reliably. Env var inheritance through msbuild.sh/tools.sh + # is unreliable for this property. + if args.ios_rid: + send_params.append(f"/p:iOSRid={args.ios_rid}") + run_msbuild_command(send_params, warn_as_error=False) diff --git a/src/scenarios/mauiiosinnerloop/.gitignore b/src/scenarios/mauiiosinnerloop/.gitignore new file mode 100644 index 00000000000..cd91a583afb --- /dev/null +++ b/src/scenarios/mauiiosinnerloop/.gitignore @@ -0,0 +1,4 @@ +# Local measurement output produced by run-local.sh — never commit. +# (run-local.sh copies traces/*.binlog and *-versions.json into results// +# for local inspection. Production results go to the perf-lab via upload.py.) +results/ diff --git a/src/scenarios/mauiiosinnerloop/post.py b/src/scenarios/mauiiosinnerloop/post.py new file mode 100644 index 00000000000..c2d8dc0802a --- /dev/null +++ b/src/scenarios/mauiiosinnerloop/post.py @@ -0,0 +1,52 @@ +''' +post cleanup script +''' + +import os +import subprocess +import sys +import traceback +from performance.logger import setup_loggers, getLogger +from shared.ioshelper import iOSHelper +from shared.postcommands import clean_directories +from test import EXENAME + +setup_loggers(True) +logger = getLogger(__name__) + +try: + bundle_id = f'com.companyname.{EXENAME.lower()}' + ios_rid = os.environ.get('IOS_RID', 'iossimulator-arm64') + is_physical = (ios_rid == 'ios-arm64') + + helper = iOSHelper() + try: + if is_physical: + device_udid = iOSHelper.detect_connected_device() + if device_udid: + helper.setup_device(bundle_id, None, device_udid, is_physical=True) + helper.cleanup() + else: + logger.warning("No device UDID available — skipping uninstall") + else: + helper.setup_device(bundle_id, None, 'booted', is_physical=False) + helper.cleanup() + except Exception as e: + logger.warning("iOS uninstall failed (continuing): %s", e) + + workitem_root = os.environ.get('HELIX_WORKITEM_ROOT', '') + sim_udid_path = os.path.join(workitem_root, 'sim_udid.txt') if workitem_root else '' + if sim_udid_path and os.path.isfile(sim_udid_path): + try: + with open(sim_udid_path, 'r', encoding='utf-8') as f: + created_udid = f.read().strip() + if created_udid: + logger.info("Deleting per-workitem simulator: %s", created_udid) + iOSHelper.delete_simulator(created_udid) + except Exception as e: + logger.warning("Could not delete per-workitem simulator: %s", e) + + subprocess.run(['dotnet', 'build-server', 'shutdown'], check=False) + clean_directories() +except Exception as e: + logger.warning(f"Post cleanup encountered an error (best-effort, continuing): {e}\n{traceback.format_exc()}") diff --git a/src/scenarios/mauiiosinnerloop/pre.py b/src/scenarios/mauiiosinnerloop/pre.py new file mode 100644 index 00000000000..350ac1e5f11 --- /dev/null +++ b/src/scenarios/mauiiosinnerloop/pre.py @@ -0,0 +1,523 @@ +''' +pre-command: Set up a MAUI iOS app for deploy measurement. +Creates the template (without restore) and prepares the modified file for incremental deploy. +NuGet packages are restored on the Helix machine, not shipped in the payload. +''' +import glob +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import urllib.request +import xml.etree.ElementTree as ET +import zipfile +from performance.common import get_repo_root_path +from performance.logger import setup_loggers, getLogger +from shared import const +from shared.mauisharedpython import install_latest_maui, MauiNuGetConfigContext +from shared.precommands import PreCommands +from test import EXENAME + +def install_maui_ios_workload(precommands: PreCommands): + ''' + Install the maui-ios workload (not the full 'maui' workload). + The full 'maui' workload includes Android/Windows components that aren't + needed for this scenario. Since this scenario only needs iOS, 'maui-ios' + is sufficient and much smaller. + + Uses the shared install_latest_maui() for rollback file creation and + workload install. Falls back to a manifest-patching workaround when + the iOS workload manifest references net10.0 cross-targeting packs + that don't exist on any NuGet feed (upstream coherency issue). + ''' + logger.info("########## Installing maui-ios workload ##########") + + if precommands.has_workload: + logger.info("Skipping maui-ios installation due to --has-workload=true") + return + + # Try the standard shared install path first. + try: + install_latest_maui( + precommands, + workloads=["microsoft.net.sdk.ios"], + workload_id='maui-ios', + ) + logger.info("########## Finished installing maui-ios workload ##########") + return + except subprocess.CalledProcessError as e: + logger.warning( + f"Standard workload install failed: {e}. " + "This typically happens when the iOS workload manifest (e.g. v26.4) " + "declares net10.0 cross-targeting packs that don't exist on any NuGet feed. " + "Falling back to manifest-patching workaround." + ) + + # ── Manifest-patching fallback ── + # The iOS workload manifest can reference net10.0 cross-targeting packs that + # haven't been published yet (upstream coherency issue). We work around this + # by downloading the manifest nupkg, patching out net10.0 entries, placing the + # patched manifest on disk, and installing with --skip-manifest-update. + logger.info("########## Starting manifest-patching fallback ##########") + + # Read package info from rollback_maui.json (created by install_latest_maui) + with open("rollback_maui.json", "r", encoding="utf-8") as f: + rollback_data = json.load(f) + ios_rollback = rollback_data.get("microsoft.net.sdk.ios", "") + if not ios_rollback or "/" not in ios_rollback: + raise Exception( + f"rollback_maui.json has no valid microsoft.net.sdk.ios entry: {rollback_data}" + ) + package_version, sdk_band = ios_rollback.split("/", 1) + # Reconstruct the manifest package ID from the SDK band + package_id = f"Microsoft.NET.Sdk.iOS.Manifest-{sdk_band}" + + # Look up the NuGet feed that matches the sdk_band from rollback_maui.json. + # install_latest_maui() may have fallen back to a different feed (e.g., dotnet10 + # instead of dotnet11) when creating the rollback. Using the wrong feed here + # causes HTTP 404 when downloading the manifest nupkg. + sdk_major = sdk_band.split('.')[0] # e.g., "10" from "10.0.100-preview.4" + nuget_config_path = os.path.join(get_repo_root_path(), "NuGet.config") + tree = ET.parse(nuget_config_path) + package_sources = tree.getroot().find(".//packageSources") + feed = None + for add_el in package_sources.findall("add"): + if add_el.get("key", "") == f"dotnet{sdk_major}": + feed = add_el.get("value") + break + if not feed: + raise Exception( + f"No NuGet feed found for dotnet{sdk_major} in {nuget_config_path}. " + f"sdk_band={sdk_band}, rollback entry={ios_rollback}" + ) + + # Step 1: Resolve PackageBaseAddress from the NuGet v3 service index + logger.info(f"Fetching NuGet v3 service index from {feed}") + with urllib.request.urlopen(feed, timeout=60) as resp: + service_index = json.loads(resp.read().decode('utf-8')) + + package_base_url = None + for resource in service_index.get('resources', []): + if 'PackageBaseAddress' in resource.get('@type', ''): + package_base_url = resource['@id'].rstrip('/') + break + if not package_base_url: + raise Exception( + f"Could not find PackageBaseAddress resource in NuGet v3 service index at {feed}" + ) + logger.info(f"Resolved PackageBaseAddress: {package_base_url}") + + # Step 2: Download the manifest nupkg (NuGet flat container uses lowercase IDs/versions) + nupkg_url = ( + f"{package_base_url}/{package_id.lower()}/{package_version.lower()}" + f"/{package_id.lower()}.{package_version.lower()}.nupkg" + ) + logger.info(f"Downloading manifest nupkg from {nupkg_url}") + + with tempfile.TemporaryDirectory() as tmpdir: + nupkg_path = os.path.join(tmpdir, 'manifest.nupkg') + # urlretrieve doesn't support timeout — use urlopen + manual write instead. + # Explicit timeout prevents Helix jobs from hanging indefinitely on network issues. + with urllib.request.urlopen(nupkg_url, timeout=120) as resp: + with open(nupkg_path, 'wb') as dl_file: + dl_file.write(resp.read()) + logger.info(f"Downloaded manifest nupkg to {nupkg_path}") + + # Step 3: Extract all files from the data/ directory inside the nupkg + extracted_files = {} + with zipfile.ZipFile(nupkg_path, 'r') as zf: + for entry in zf.namelist(): + if entry.startswith('data/') and not entry.endswith('/'): + filename = os.path.basename(entry) + extracted_files[filename] = zf.read(entry) + logger.info(f"Extracted from nupkg: {entry} ({len(extracted_files[filename])} bytes)") + + if 'WorkloadManifest.json' not in extracted_files: + raise Exception( + f"WorkloadManifest.json not found in data/ directory of {nupkg_url}. " + f"Found files: {list(extracted_files.keys())}" + ) + + # Step 4: Patch WorkloadManifest.json — remove entries containing "net10" + raw_text = extracted_files['WorkloadManifest.json'].decode('utf-8') + # The manifest uses trailing commas (invalid JSON). Strip them before parsing. + # Python's json.loads() (especially 3.14+) rejects trailing commas. + cleaned_text = re.sub(r',\s*([}\]])', r'\1', raw_text) + manifest = json.loads(cleaned_text) + + removed_packs = [] + removed_extends = [] + removed_top_packs = [] + + # Patch workloads.ios.packs — remove entries containing "net10" (case-insensitive). + # The packs field can be a list (of pack name strings) or a dict (pack names as keys), + # depending on the manifest version. + ios_workload = manifest.get('workloads', {}).get('ios', {}) + if 'packs' in ios_workload: + if isinstance(ios_workload['packs'], list): + original_items = list(ios_workload['packs']) + ios_workload['packs'] = [ + item for item in ios_workload['packs'] + if 'net10' not in item.lower() + ] + removed_packs = [item for item in original_items if 'net10' in item.lower()] + else: + original_keys = list(ios_workload['packs'].keys()) + for key in original_keys: + if 'net10' in key.lower(): + del ios_workload['packs'][key] + removed_packs.append(key) + + # Patch workloads.ios.extends — remove list items containing "net10" + if 'extends' in ios_workload: + original_extends = list(ios_workload['extends']) + ios_workload['extends'] = [ + item for item in ios_workload['extends'] + if 'net10' not in item.lower() + ] + removed_extends = [ + item for item in original_extends + if 'net10' in item.lower() + ] + + # Patch top-level packs — remove keys containing "net10". + # This should always be a dict, but guard against unexpected list format. + if 'packs' in manifest: + if isinstance(manifest['packs'], dict): + original_keys = list(manifest['packs'].keys()) + for key in original_keys: + if 'net10' in key.lower(): + del manifest['packs'][key] + removed_top_packs.append(key) + else: + logger.warning( + f"Top-level 'packs' is {type(manifest['packs']).__name__}, expected dict — " + "skipping top-level packs patching" + ) + + logger.info( + f"Patched WorkloadManifest.json — removed: " + f"workloads.ios.packs={removed_packs}, " + f"workloads.ios.extends={removed_extends}, " + f"top-level packs={removed_top_packs}" + ) + + if not removed_packs and not removed_extends and not removed_top_packs: + logger.warning( + "Manifest patching removed NO entries — the install failure may not be " + "caused by missing net10 packs. The subsequent install will likely fail " + "with the same error." + ) + + # json.dumps produces valid JSON (no trailing commas) — the SDK accepts both. + patched_json = json.dumps(manifest, indent=2) + extracted_files['WorkloadManifest.json'] = patched_json.encode('utf-8') + + # Step 5: Determine DOTNET_ROOT + dotnet_root = os.environ.get('DOTNET_ROOT') + if not dotnet_root: + dotnet_path = shutil.which('dotnet') + if not dotnet_path: + raise Exception("Cannot determine DOTNET_ROOT: not set and dotnet not found in PATH") + dotnet_root = os.path.dirname(os.path.realpath(dotnet_path)) + logger.info(f"DOTNET_ROOT: {dotnet_root}") + + # Step 6: Place patched manifest files on disk + # sdk_band comes from the manifest package ID (e.g., "11.0.100-preview.4") + # package_version is the NuGet version (e.g., "26.4.11427-net11-p4") + target_dir = os.path.join( + dotnet_root, 'sdk-manifests', sdk_band, + 'microsoft.net.sdk.ios', package_version + ) + os.makedirs(target_dir, exist_ok=True) + logger.info(f"Writing patched manifest files to {target_dir}") + + for filename, content in extracted_files.items(): + target_path = os.path.join(target_dir, filename) + with open(target_path, 'wb') as f: + f.write(content) + logger.info(f"Wrote {target_path} ({len(content)} bytes)") + + # Step 7: Install with --skip-manifest-update (manifest is already on disk) + logger.info("Installing maui-ios workload with --skip-manifest-update (patched manifest)") + precommands.install_workload('maui-ios', ['--skip-manifest-update']) + logger.info("########## Finished installing maui-ios workload (manifest-patched) ##########") + +def check_xcode_compatibility(framework: str): + ''' + Best-effort check that the active Xcode version matches the iOS SDK's + _RecommendedXcodeVersion. Logs a warning on mismatch — does not fail. + The caller (pipeline or run-local.sh) handles the actual Xcode selection. + ''' + try: + result = subprocess.run( + ['xcodebuild', '-version'], + capture_output=True, text=True, timeout=10 + ) + if result.returncode != 0: + logger.warning("Could not detect Xcode version (xcodebuild -version failed)") + return + + # Parse "Xcode 26.4" → "26.4" + xcode_line = result.stdout.strip().split('\n')[0] + xcode_match = re.search(r'Xcode\s+(\d+\.\d+)', xcode_line) + if not xcode_match: + logger.warning(f"Could not parse Xcode version from: {xcode_line}") + return + active_xcode = xcode_match.group(1) + + # Extract TFM prefix: "net11.0-ios" → "net11.0" + tfm_prefix = framework.split('-')[0] if '-' in framework else framework + + # Find the iOS SDK Versions.props file + dotnet_path = shutil.which('dotnet') + if not dotnet_path: + logger.warning("Could not find dotnet in PATH — skipping Xcode version check") + return + dotnet_dir = os.path.dirname(os.path.realpath(dotnet_path)) + packs_dir = os.path.join(dotnet_dir, 'packs') + + # Search for Microsoft.iOS.Sdk._*/*/targets/Microsoft.iOS.Sdk.Versions.props + search_pattern = os.path.join( + packs_dir, + f'Microsoft.iOS.Sdk.{tfm_prefix}_*', + '*', 'targets', 'Microsoft.iOS.Sdk.Versions.props' + ) + props_files = sorted(glob.glob(search_pattern)) + if not props_files: + logger.warning( + f"Could not find Microsoft.iOS.Sdk.Versions.props for {tfm_prefix} " + f"in {packs_dir} — skipping Xcode version check" + ) + return + + # Use the last (highest version) match + versions_props = props_files[-1] + logger.info(f"Found iOS SDK Versions.props: {versions_props}") + + # Parse _RecommendedXcodeVersion + with open(versions_props, 'r') as f: + content = f.read() + rec_match = re.search(r'<_RecommendedXcodeVersion>([^<]+)', content) + if not rec_match: + logger.warning(f"Could not find _RecommendedXcodeVersion in {versions_props}") + return + required_xcode = rec_match.group(1).strip() + + active_major_minor = '.'.join(active_xcode.split('.')[:2]) + required_major_minor = '.'.join(required_xcode.split('.')[:2]) + + if active_major_minor != required_major_minor: + logger.warning( + f"Xcode version MISMATCH: " + f"active Xcode is {active_xcode} but iOS SDK requires {required_xcode}. " + f"The build may fail with _ValidateXcodeVersion error. " + f"Set XCODE_PATH or DEVELOPER_DIR to a compatible Xcode installation." + ) + else: + logger.info( + f"Xcode version OK: active={active_xcode}, " + f"required={required_xcode} (major.minor match: {active_major_minor})" + ) + except Exception as e: + logger.warning(f"Xcode compatibility check failed (non-fatal): {e}") + +def strip_non_ios_tfms(csproj_path: str, framework: str): + ''' + Strip non-iOS TargetFrameworks from the generated .csproj. + The MAUI template (since .NET 10+) generates multiple conditional + elements for android, ios, maccatalyst, and windows. + We replace all of them with a single unconditional + containing only the iOS TFM we want to build. + Uses ]*> to match both unconditional and + Condition="..." variants of the element. + ''' + with open(csproj_path, 'r') as f: + content = f.read() + + logger.info(f"Stripping non-iOS TFMs from {csproj_path}, keeping: {framework}") + + # Remove all existing ... lines + # (both unconditional and conditional variants). + stripped = re.sub( + r'\s*]*>[^<]*', + '', + content + ) + + # Also handle singular if present + stripped = re.sub( + r'\s*]*>[^<]*', + '', + stripped + ) + + # Insert a single unconditional with the iOS TFM + # into the first + stripped = stripped.replace( + '', + f'\n {framework}', + 1 # only the first PropertyGroup + ) + + with open(csproj_path, 'w') as f: + f.write(stripped) + + logger.info(f"Stripped non-iOS TFMs. csproj now targets: {framework}") + +def inject_csproj_properties(csproj_path: str, properties: dict): + '''Inject MSBuild properties into the first PropertyGroup of a csproj.''' + with open(csproj_path, 'r') as f: + content = f.read() + + if '' not in content: + raise Exception(f"No found in {csproj_path}") + + for name, value in properties.items(): + if f'<{name}>' not in content: + content = content.replace( + '', + f' <{name}>{value}\n ', + 1 + ) + + with open(csproj_path, 'w') as f: + f.write(content) + logger.info(f"Injected properties into {csproj_path}: {list(properties.keys())}") + +setup_loggers(True) +logger = getLogger(__name__) +logger.info("Starting pre-command for MAUI iOS deploy measurement") + +precommands = PreCommands() + +with MauiNuGetConfigContext(precommands.framework): + install_maui_ios_workload(precommands) + check_xcode_compatibility(precommands.framework) + precommands.print_dotnet_info() + + # Create template without restoring packages — packages will be restored + # on the Helix machine to avoid shipping ~1-2GB in the workitem payload. + precommands.new(template='maui', + output_dir=const.APPDIR, + bin_dir=const.BINDIR, + exename=EXENAME, + working_directory=sys.path[0], + no_restore=True) + + # Copy the merged NuGet.config into the app directory. This file contains + # MAUI NuGet feed URLs added by MauiNuGetConfigContext. The Helix machine + # needs these feeds during restore, and we must copy before the context + # manager restores the original NuGet.config. + repo_root = os.path.normpath(os.path.join(sys.path[0], '..', '..', '..')) + repo_nuget_config = os.path.join(repo_root, 'NuGet.config') + app_nuget_config = os.path.join(const.APPDIR, 'NuGet.config') + shutil.copy2(repo_nuget_config, app_nuget_config) + logger.info(f"Copied merged NuGet.config from {repo_nuget_config} to {app_nuget_config}") + + # Prepare the csproj: strip non-iOS TFMs and inject required properties + csproj_path = os.path.join(const.APPDIR, f'{EXENAME}.csproj') + strip_non_ios_tfms(csproj_path, precommands.framework) + inject_csproj_properties(csproj_path, { + # Preview SDKs may lack prune-package-data files, causing NETSDK1226. + 'AllowMissingPrunePackageData': 'true', + # Re-enable Roslyn compiler server (perf repo disables it globally + # for BenchmarkDotNet) to match real developer inner loop. + 'UseSharedCompilation': 'true', + }) + + # Create modified source files in src/ for the incremental deploy simulation. + # The runner toggles between original and modified versions each iteration, + # exercising both the C# compiler (Csc) and XAML compiler (XamlC) paths. + src_dir = os.path.join(sys.path[0], const.SRCDIR) + os.makedirs(src_dir, exist_ok=True) + + # --- Modified MainPage.xaml.cs: add a debug line in the constructor --- + # The template may place MainPage in either the root or Pages/ subdirectory. + # Normalize to ALWAYS use Pages/ so that the hardcoded --edit-dest paths in + # maui_scenarios_ios_innerloop.proj ("app/Pages/MainPage.xaml.cs") are valid. + pages_dir = os.path.join(const.APPDIR, 'Pages') + cs_candidates = [ + os.path.join(const.APPDIR, 'Pages', 'MainPage.xaml.cs'), + os.path.join(const.APPDIR, 'MainPage.xaml.cs'), + ] + cs_original = None + for candidate in cs_candidates: + if os.path.exists(candidate): + cs_original = candidate + break + if cs_original is None: + raise Exception( + "Could not find MainPage.xaml.cs in template — " + f"searched: {cs_candidates}" + ) + + # If MainPage files are at the root, move them into Pages/ so that the + # .proj's hardcoded edit-dest paths are always correct. + if os.path.dirname(os.path.abspath(cs_original)) != os.path.abspath(pages_dir): + os.makedirs(pages_dir, exist_ok=True) + for fname in ['MainPage.xaml.cs', 'MainPage.xaml']: + src_file = os.path.join(const.APPDIR, fname) + if os.path.exists(src_file): + shutil.move(src_file, os.path.join(pages_dir, fname)) + logger.info(f"Moved {src_file} → {os.path.join(pages_dir, fname)}") + cs_original = os.path.join(pages_dir, 'MainPage.xaml.cs') + + cs_modified = os.path.join(src_dir, 'MainPage.xaml.cs') + with open(cs_original, 'r') as f: + cs_content = f.read() + + cs_modified_content = cs_content.replace( + 'InitializeComponent();', + 'InitializeComponent();\n\t\tSystem.Diagnostics.Debug.WriteLine("incremental-touch");' + ) + if cs_modified_content == cs_content: + raise Exception( + "Could not find 'InitializeComponent();' in %s — template may have changed" % cs_original + ) + + with open(cs_modified, 'w') as f: + f.write(cs_modified_content) + logger.info(f"Modified MainPage.xaml.cs written to {cs_modified}") + + # --- Modified MainPage.xaml: change a label's text --- + # Look in the same directory where we found the .cs file + xaml_original = os.path.join(os.path.dirname(cs_original), 'MainPage.xaml') + if not os.path.exists(xaml_original): + raise Exception(f"Could not find MainPage.xaml at {xaml_original}") + + xaml_modified = os.path.join(src_dir, 'MainPage.xaml') + with open(xaml_original, 'r') as f: + xaml_content = f.read() + + # Use a flexible match — look for any Text="..." attribute on a Label + # to handle template variations. Prefer a known string first. + xaml_modified_content = xaml_content.replace( + 'Text="Hello, World!"', + 'Text="Hello, World! (updated)"' + ) + if xaml_modified_content == xaml_content: + # Fallback: try the .NET 10+ template's "Task Categories" text + xaml_modified_content = xaml_content.replace( + 'Text="Task Categories"', + 'Text="Task Categories (updated)"' + ) + if xaml_modified_content == xaml_content: + # Last resort: replace the first Text="..." attribute we find + xaml_modified_content = re.sub( + r'Text="([^"]*)"', + r'Text="\1 (updated)"', + xaml_content, + count=1 + ) + if xaml_modified_content == xaml_content: + raise Exception( + "Could not find any Text=\"...\" attribute in %s — template may have changed" % xaml_original + ) + + with open(xaml_modified, 'w') as f: + f.write(xaml_modified_content) + logger.info(f"Modified MainPage.xaml written to {xaml_modified}") diff --git a/src/scenarios/mauiiosinnerloop/select_xcode.py b/src/scenarios/mauiiosinnerloop/select_xcode.py new file mode 100644 index 00000000000..9379918ab21 --- /dev/null +++ b/src/scenarios/mauiiosinnerloop/select_xcode.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""select_xcode.py — Select the Xcode matching the iOS SDK version. + +Standalone (no repo imports). Prints selected path to stdout; logs to stderr. +""" +import argparse, glob, json, os, re, subprocess, sys +import xml.etree.ElementTree as ET + +_MIN_XCODE_MAJOR = 26 # Must match setup_helix.py + +def _log(msg): print(msg, file=sys.stderr) + +def _from_rollback(scenario_dir): + """Tier 1: Required Xcode major.minor from rollback_maui.json.""" + path = os.path.join(scenario_dir, "rollback_maui.json") + if not os.path.isfile(path): + return None + try: + with open(path, encoding="utf-8") as f: + ios = json.load(f).get("microsoft.net.sdk.ios", "") + # "26.2.11591-net11-p4/11.0.100-preview.3" → major=26, minor=2 + m = re.match(r"(\d+)\.(\d+)", ios.split("/")[0]) + return (int(m.group(1)), int(m.group(2))) if m else None + except (json.JSONDecodeError, OSError): + return None + +def _from_sdk_packs(dotnet_root): + """Tier 2: Required Xcode major.minor from SDK Versions.props.""" + if not dotnet_root: + return None + pattern = os.path.join(dotnet_root, "packs", "Microsoft.iOS.Sdk.*", + "*", "targets", "Microsoft.iOS.Sdk.Versions.props") + for props in sorted(glob.glob(pattern), reverse=True): + try: + for elem in ET.parse(props).iter("_RecommendedXcodeVersion"): + m = re.match(r"(\d+)\.(\d+)", elem.text or "") + if m: + return (int(m.group(1)), int(m.group(2))) + except ET.ParseError: + continue + return None + +def _parse_dir_version(name): + """'Xcode_26.2.app' or 'Xcode-26.2.0.app' → (26, 2) or (26, 2, 0).""" + stem = re.sub(r"^Xcode[_-]", "", name.replace(".app", "")) + try: + return tuple(int(p) for p in stem.split(".")) + except ValueError: + return None + +def _find_xcode(required): + """Find best Xcode in /Applications matching required (major, minor). + If required is None, picks the highest version >= _MIN_XCODE_MAJOR. + Also considers /Applications/Xcode.app via its embedded xcodebuild.""" + candidates = [] + for entry in sorted(os.listdir("/Applications")): + full = os.path.join("/Applications", entry) + if not (entry.endswith(".app") and os.path.isdir(full)): + continue + # Handles Xcode_26.2.app (Helix) and Xcode-26.2.0.app (Xcodes.app) + if re.match(r"Xcode[_-]\d+", entry): + ver = _parse_dir_version(entry) + elif entry == "Xcode.app": + try: + xb = os.path.join(full, "Contents/Developer/usr/bin/xcodebuild") + out = subprocess.run([xb, "-version"], capture_output=True, + text=True, timeout=10).stdout + m = re.search(r"Xcode\s+(\d+)\.(\d+)", out) + ver = (int(m.group(1)), int(m.group(2))) if m else None + except (OSError, subprocess.TimeoutExpired): + continue + else: + continue + if ver and len(ver) >= 2: + ok = (ver[0] == required[0] and ver[1] == required[1]) if required \ + else (ver[0] >= _MIN_XCODE_MAJOR) + if ok: + candidates.append((full, ver)) + candidates.sort(key=lambda x: x[1]) + return candidates[-1][0] if candidates else None + +if __name__ == "__main__": + p = argparse.ArgumentParser(description="Select Xcode matching iOS SDK") + p.add_argument("--scenario-dir", default=os.path.dirname(os.path.abspath(__file__))) + p.add_argument("--dotnet-root", default=os.environ.get("DOTNET_ROOT", "")) + args = p.parse_args() + + req = _from_rollback(args.scenario_dir) + if req: + _log(f"Required Xcode {req[0]}.{req[1]} (from rollback_maui.json)") + else: + req = _from_sdk_packs(args.dotnet_root) + if req: + _log(f"Required Xcode {req[0]}.{req[1]} (from SDK packs)") + else: + _log(f"No version constraint; selecting highest >= {_MIN_XCODE_MAJOR}") + selected = _find_xcode(req) + if not selected and req: + _log(f"No Xcode matching {req[0]}.{req[1]}; trying any >= {_MIN_XCODE_MAJOR}") + selected = _find_xcode(None) + if not selected: + _log("ERROR: No suitable Xcode found in /Applications/") + sys.exit(1) + _log(f"Selected: {selected}") + print(selected) diff --git a/src/scenarios/mauiiosinnerloop/setup_helix.py b/src/scenarios/mauiiosinnerloop/setup_helix.py new file mode 100644 index 00000000000..6983fb4b1df --- /dev/null +++ b/src/scenarios/mauiiosinnerloop/setup_helix.py @@ -0,0 +1,1324 @@ +#!/usr/bin/env python3 +"""setup_helix.py — Helix machine setup for MAUI iOS inner loop (macOS). + +Runs on the Helix machine BEFORE test.py. Bootstraps the macOS environment +for iOS builds: + 1. Configure DOTNET_ROOT and PATH from the correlation payload SDK. + 2. Select the Xcode version that matches the iOS SDK workload packs. + 3. Validate iOS simulator runtime availability. + 4. Boot the target iOS simulator device. + 5. (Device only) Locate signing artifacts; FAIL the work item if missing. + 6. Install the maui-ios workload. + 7. Restore NuGet packages for the app project. + 8. Disable Spotlight indexing on the workitem directory. + +Xcode selection strategy: The iOS SDK packs require a SPECIFIC Xcode version +(e.g. packs 26.2.x need Xcode 26.2). This script derives the required Xcode +major.minor from the ``rollback_maui.json`` file created by pre.py (shipped in +the workitem payload), then selects a matching ``/Applications/Xcode_*.app``. +If rollback_maui.json is absent or unparseable, it falls back to a coarse +``>= _MIN_XCODE_MAJOR`` check. This fails the work item early instead of +wasting 20+ minutes on workload install before hitting _ValidateXcodeVersion. + +Device path & infrastructure prerequisites +------------------------------------------ +For the physical-device variant (IOS_RID=ios-arm64) the build runs with +``EnableCodeSigning=false`` to keep MSBuild deterministic on Helix; the +post-build ``ioshelper.sign_app_for_device`` re-signs the .app using: + + - ``embedded.mobileprovision`` — staged into HELIX_WORKITEM_ROOT (CWD) + - ``sign`` tool — symlinked into the venv ``bin/`` so it's on PATH + +Both must be present somewhere on the Helix machine (see +``_SIGNING_SEARCH_ROOTS``). The Mac.iPhone.17.Perf queue had Helix machine +prep install them; newer queues like Mac.iPhone.13.Perf currently do NOT, +which is a tracked machine-image gap. When the artifacts are missing, +``find_and_stage_signing_artifacts`` returns False and the work item +FAILS LOUDLY with sys.exit(1) and a ``WORK ITEM FAILED — DEVICE INFRA +UNAVAILABLE`` banner in the console log. We deliberately do NOT mask +the failure as a "skip" / pass: a green build must mean the scenario +actually ran, not that we silently sidestepped a queue gap. The fix is +to provision the queue (Engineering Services ticket), not to flip a +flag in this script. +""" + +import json +import os +import platform +import re +import subprocess +import sys +from datetime import datetime + +# --- Logging --- +# Follows the same logging pattern as the Android inner loop setup_helix.py: +# structured log file written to HELIX_WORKITEM_UPLOAD_ROOT for post-mortem +# debugging, with key messages also printed to stdout for Helix console output. +_logfile = None + + +def log(msg, tee=False): + """Write *msg* with a timestamp to the log file.""" + line = f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {msg}" + if _logfile: + _logfile.write(line + "\n") + _logfile.flush() + if tee: + print(line, flush=True) + + +def log_raw(msg, tee=False): + """Write *msg* verbatim (no timestamp) to the log file.""" + if _logfile: + _logfile.write(msg + "\n") + _logfile.flush() + if tee: + print(msg, flush=True) + + +def run_cmd(args, check=True, **kwargs): + """Run a command, logging stdout/stderr. Returns CompletedProcess.""" + log(f"Running: {args}") + result = subprocess.run( + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + **kwargs, + ) + if result.stdout: + for line in result.stdout.splitlines(): + log_raw(line) + if check and result.returncode != 0: + raise subprocess.CalledProcessError(result.returncode, args, result.stdout) + return result + + +def _dump_log(): + """Print the full log file to stdout so it appears in Helix console output.""" + if not _logfile: + return + _logfile.flush() + try: + with open(_logfile.name, "r") as f: + print(f.read()) + except Exception: + pass + + +# --- Setup Steps --- + +def print_diagnostics(): + """Log environment variables useful for debugging Helix failures.""" + log_raw("=== DIAGNOSTICS ===", tee=True) + for var in ["DOTNET_ROOT", "PATH", "DEVELOPER_DIR", "HELIX_WORKITEM_ROOT", + "HELIX_CORRELATION_PAYLOAD", "TMPDIR", "USER", "HOME"]: + log_raw(f" {var}={os.environ.get(var, '')}") + log_raw(f" macOS version:", tee=True) + run_cmd(["sw_vers"], check=False) + log_raw(f" whoami / id:", tee=True) + run_cmd(["id"], check=False) + + +# CoreSimulator folders that may need to be owned by the current user before +# 'simctl boot' can succeed. On shared Helix machines, these folders may +# have accumulated state owned by a previous tenant (root, ado_agent, etc.), +# which causes 'simctl boot' to fail with NSCocoaErrorDomain code 513 +# ("You don't have permission to save the file ... in the folder +# CoreSimulator") — even for simulators we just created ourselves, because +# the boot writes log/state files into shared CoreSimulator folders we +# don't own. +_CORESIMULATOR_PATHS = [ + "/Library/Developer/CoreSimulator", + "/Library/Logs/CoreSimulator", + "~/Library/Developer/CoreSimulator", + "~/Library/Logs/CoreSimulator", + "~/Library/Caches/com.apple.CoreSimulator.SimulatorTrampoline", + "~/Library/Saved Application State/com.apple.CoreSimulator.CoreSimulatorService.savedState", +] + +# Subdirectory names underneath /Library/Developer/CoreSimulator that are +# Apple-managed read-only content (mounted runtime images, device-type +# bundles, signed system cryptexes). We have no permission to chown them and +# never need to — `simctl boot` only writes to Devices/ and Logs/. Without +# pruning these names, a recursive chown on /Library/Developer/CoreSimulator +# walks the iOS runtime volume (e.g. .../Volumes/iOS_23E254a/) and emits one +# "Operation not permitted" line per file — ~700k lines / 200+ MB of console +# log spam per Helix work item. +_CORESIMULATOR_PRUNE_NAMES = ("Volumes", "Profiles", "Cryptex", "Images") + + +def _sudo_chown_pruning(path, owner): + """``sudo chown -R owner path`` but prune Apple read-only subtrees. + + Equivalent to:: + + sudo find \\( -name Volumes -o -name Profiles -o + -name Cryptex -o -name Images \\) -prune \\ + -o -exec chown owner {} + + + Used in place of plain ``chown -R`` for the system-wide CoreSimulator + paths (see ``_CORESIMULATOR_PRUNE_NAMES`` for why). The plain ``chown -R`` + walks the iOS runtime image and emits hundreds of thousands of + "Operation not permitted" lines we then dutifully copy into the Helix + log; this variant skips those read-only mount points at the source. + """ + name_clause: list[str] = [] + for name in _CORESIMULATOR_PRUNE_NAMES: + if name_clause: + name_clause.append("-o") + name_clause.extend(["-name", name]) + cmd = ( + ["sudo", "find", path, "("] + name_clause + [")", "-prune", + "-o", "-exec", "chown", owner, "{}", "+"] + ) + return run_cmd(cmd, check=False) + + +def fix_coresimulator_permissions(): + """Take ownership of CoreSimulator folders so simctl boot can succeed. + + Helix machines are shared; CoreSimulator folders may have accumulated + state owned by prior tenants. ``simctl boot`` writes log/state files + into several Library folders, and even a freshly-created device fails + to boot if any one of them is not writable by the current user. This + function chowns the relevant folders to the current user so that boot, + spawn, and shutdown all work. + + For the system-wide ``/Library/Developer/CoreSimulator`` tree we use + ``find ... -prune`` rather than a plain ``chown -R`` to avoid recursing + into Apple's read-only runtime/profile/cryptex mount points (see + ``_CORESIMULATOR_PRUNE_NAMES``). For per-user paths under ``~/Library`` + we use plain ``chown -R`` because they don't contain those mounts. + + It is best-effort: paths that don't exist are skipped, and chown + failures are logged as warnings so the rest of setup still runs (the + boot itself will fail loudly with diagnostics if permissions are still + wrong). + """ + log_raw("=== COREsimulator PERMISSIONS ===", tee=True) + user = os.environ.get("USER") or "" + if not user: + # Fallback: query the OS + result = run_cmd(["whoami"], check=False) + user = (result.stdout or "").strip() + if not user: + log("WARNING: Cannot determine current user; skipping CoreSimulator chown", + tee=True) + return + owner = f"{user}:staff" + + for raw in _CORESIMULATOR_PATHS: + path = os.path.expanduser(raw) + if not os.path.exists(path): + log(f" skip (not present): {path}") + continue + # Only the system-wide /Library/Developer/CoreSimulator tree contains + # the Apple-mounted read-only volumes that explode chown -R output. + if path == "/Library/Developer/CoreSimulator": + log(f" sudo find {path} (prune {_CORESIMULATOR_PRUNE_NAMES}) " + f"-exec chown {owner}", tee=True) + result = _sudo_chown_pruning(path, owner) + else: + log(f" sudo chown -R {owner} {path}", tee=True) + result = run_cmd(["sudo", "chown", "-R", owner, path], check=False) + if result.returncode != 0: + log(f"WARNING: chown on {path} failed (exit {result.returncode}). " + f"simctl boot may still hit permission errors.", tee=True) + # Show ownership for diagnostics (top-level only — ls -la is non-recursive) + run_cmd(["ls", "-la", path], check=False) + + +def setup_dotnet(correlation_payload): + """Set DOTNET_ROOT and PATH to point at the SDK in the correlation payload. + + Returns the path to the dotnet executable. + """ + dotnet_root = os.path.join(correlation_payload, "dotnet") + dotnet_exe = os.path.join(dotnet_root, "dotnet") + + os.environ["DOTNET_ROOT"] = dotnet_root + os.environ["PATH"] = dotnet_root + ":" + os.environ.get("PATH", "") + + log(f"DOTNET_ROOT={dotnet_root}", tee=True) + run_cmd([dotnet_exe, "--version"], check=False) + return dotnet_exe + + +# Coarse pre-filter: packs are not yet installed at this point (workload +# install happens at step 5), so we can't read _RecommendedXcodeVersion from +# the pack's Versions.props. This >= 26 check catches clearly incompatible +# machines early. The SDK's _ValidateXcodeVersion target performs the exact +# version check at build time. +_MIN_XCODE_MAJOR = 26 + + +def _parse_xcode_version(output): + """Parse the major.minor version tuple from ``xcodebuild -version`` output. + + Expects a line like "Xcode 26.2" or "Xcode 15.0.1". + Returns (major, minor) as ints, or None if parsing fails. + """ + m = re.search(r"Xcode\s+(\d+)\.(\d+)", output or "") + if m: + return int(m.group(1)), int(m.group(2)) + return None + + +def _get_required_xcode_version(workitem_root): + """Derive the required Xcode major.minor from rollback_maui.json. + + The rollback file is created by pre.py and shipped in the workitem payload. + It contains e.g.:: + + {"microsoft.net.sdk.ios": "26.2.11591-net11-p4/11.0.100-preview.3"} + + The version prefix ``26.2`` IS the required Xcode version (major.minor). + Returns ``(major, minor)`` as ints, or ``None`` if the file is absent or + unparseable. + """ + rollback_path = os.path.join(workitem_root, "rollback_maui.json") + if not os.path.isfile(rollback_path): + log("rollback_maui.json not found — cannot derive required Xcode version") + return None + + try: + with open(rollback_path, "r", encoding="utf-8") as f: + data = json.load(f) + except (json.JSONDecodeError, OSError) as e: + log(f"WARNING: Failed to read rollback_maui.json: {e}") + return None + + ios_value = data.get("microsoft.net.sdk.ios") + if not ios_value: + log("WARNING: rollback_maui.json has no 'microsoft.net.sdk.ios' key") + return None + + # Value format: "26.2.11591-net11-p4/11.0.100-preview.3" + # Extract version prefix before the "/" (band), then parse major.minor + version_part = ios_value.split("/")[0] # "26.2.11591-net11-p4" + m = re.match(r"(\d+)\.(\d+)", version_part) + if not m: + log(f"WARNING: Could not parse major.minor from iOS SDK version '{version_part}'") + return None + + major, minor = int(m.group(1)), int(m.group(2)) + log(f"Required Xcode version from rollback_maui.json: {major}.{minor} " + f"(from '{ios_value}')", tee=True) + return (major, minor) + + +def _parse_xcode_dir_version(dirname): + """Parse version components from an Xcode directory name. + + ``"Xcode_26.2.app"`` → ``(26, 2)``, ``"Xcode_26.2.1.app"`` → ``(26, 2, 1)``. + Returns a tuple of ints, or ``None`` if parsing fails. + """ + # Strip "Xcode_" prefix and ".app" suffix, then split on "." + stem = dirname.replace("Xcode_", "").replace(".app", "") + parts = stem.split(".") + try: + return tuple(int(p) for p in parts) + except ValueError: + return None + + +def select_xcode(workitem_root): + """Select the Xcode version that matches the iOS SDK workload packs. + + Derives the required Xcode major.minor from rollback_maui.json (created by + pre.py). If the system-default Xcode already matches, no switching is + needed. Otherwise, searches /Applications/Xcode_*.app for a matching + version and activates it via ``sudo xcode-select -s``. Falls back to the + coarse ``>= _MIN_XCODE_MAJOR`` check when rollback_maui.json is absent. + """ + log_raw("=== XCODE SELECTION ===", tee=True) + + required = _get_required_xcode_version(workitem_root) + + # Log the current default before any changes + run_cmd(["xcode-select", "-p"], check=False) + result = run_cmd(["xcodebuild", "-version"], check=False) + + current = _parse_xcode_version(result.stdout) + if current is None: + log("ERROR: Could not parse Xcode version from xcodebuild output. " + "Ensure Xcode is installed.", tee=True) + _dump_log() + sys.exit(1) + + cur_major, cur_minor = current + log(f"Default Xcode version: {cur_major}.{cur_minor}", tee=True) + + # --- Precise matching mode (rollback file provides required version) --- + if required is not None: + req_major, req_minor = required + + if cur_major == req_major and cur_minor == req_minor: + log(f"Default Xcode {cur_major}.{cur_minor} matches required " + f"{req_major}.{req_minor} — no switching needed.", tee=True) + return + + # Default doesn't match — search for a matching Xcode_*.app + log(f"Default Xcode {cur_major}.{cur_minor} does not match required " + f"{req_major}.{req_minor}. Searching /Applications/Xcode_*.app...", + tee=True) + + matching_apps = [] + for entry in os.listdir("/Applications"): + if not (entry.startswith("Xcode_") and entry.endswith(".app")): + continue + if not os.path.isdir(os.path.join("/Applications", entry)): + continue + ver = _parse_xcode_dir_version(entry) + if ver and len(ver) >= 2 and ver[0] == req_major and ver[1] == req_minor: + matching_apps.append((entry, ver)) + + if not matching_apps: + # List all available Xcode installations for diagnostics + all_xcodes = sorted( + e for e in os.listdir("/Applications") + if e.startswith("Xcode") and e.endswith(".app") + ) + log(f"ERROR: No Xcode matching {req_major}.{req_minor} found. " + f"Available: {all_xcodes}", tee=True) + log(f"The iOS SDK packs require Xcode {req_major}.{req_minor}. " + "Install the matching Xcode or update the workload pin.", + tee=True) + _dump_log() + sys.exit(1) + + # If multiple match (e.g. Xcode_26.2.app and Xcode_26.2.1.app), + # pick the highest patch version + matching_apps.sort(key=lambda x: x[1]) + selected_name, selected_ver = matching_apps[-1] + selected = os.path.join("/Applications", selected_name) + + log(f"Found matching Xcode installations: " + f"{[name for name, _ in matching_apps]}", tee=True) + log(f"Selecting: {selected}", tee=True) + run_cmd(["sudo", "xcode-select", "-s", selected], check=False) + + # Log the new state after switching + run_cmd(["xcode-select", "-p"], check=False) + result = run_cmd(["xcodebuild", "-version"], check=False) + + version = _parse_xcode_version(result.stdout) + if version: + log(f"Xcode version after switching: {version[0]}.{version[1]}", + tee=True) + + # Validate the switch actually produced the required version + if version is None or version[0] != req_major or version[1] != req_minor: + effective = f"{version[0]}.{version[1]}" if version else "unknown" + log(f"ERROR: Xcode switch failed — active version is {effective} " + f"but {req_major}.{req_minor} is required. " + f"sudo xcode-select -s may have failed silently.", tee=True) + _dump_log() + sys.exit(1) + return + + # --- Fallback mode (no rollback file — coarse >= check) --- + log("WARNING: No rollback_maui.json — falling back to coarse " + f">= {_MIN_XCODE_MAJOR}.0 check", tee=True) + + if cur_major >= _MIN_XCODE_MAJOR: + log(f"Default Xcode {cur_major}.{cur_minor} meets minimum " + f"({_MIN_XCODE_MAJOR}.0) — no switching needed.", tee=True) + return + + # Default Xcode is too old — search for a newer Xcode_*.app + log(f"Default Xcode {cur_major}.{cur_minor} is below minimum " + f"({_MIN_XCODE_MAJOR}.0). " + "Searching for a newer /Applications/Xcode_*.app...", tee=True) + + xcode_apps = sorted( + ( + entry + for entry in os.listdir("/Applications") + if entry.startswith("Xcode_") and entry.endswith(".app") + and os.path.isdir(os.path.join("/Applications", entry)) + ), + key=lambda name: [ + int(x) for x in re.findall(r"\d+", name.replace(".app", "")) + ], + ) + + version = current + if xcode_apps: + selected = os.path.join("/Applications", xcode_apps[-1]) + log(f"Found Xcode installations: {xcode_apps}", tee=True) + log(f"Selecting highest version: {selected}", tee=True) + run_cmd(["sudo", "xcode-select", "-s", selected], check=False) + + # Log the new state after switching + run_cmd(["xcode-select", "-p"], check=False) + result = run_cmd(["xcodebuild", "-version"], check=False) + + version = _parse_xcode_version(result.stdout) + if version: + log(f"Xcode version after switching: {version[0]}.{version[1]}", + tee=True) + else: + log("WARNING: No /Applications/Xcode_*.app found.", tee=True) + + # Final validation — fail fast if still below minimum + if version is None or version[0] < _MIN_XCODE_MAJOR: + effective = f"{version[0]}.{version[1]}" if version else "unknown" + log(f"ERROR: Xcode {effective} is still below the minimum required " + f"version ({_MIN_XCODE_MAJOR}.0). The iOS SDK packs require " + f"Xcode >= {_MIN_XCODE_MAJOR}.0. Failing early to avoid wasting " + "time on workload install.", tee=True) + _dump_log() + sys.exit(1) + + +def validate_simulator_runtimes(): + """Check that iOS simulator runtimes are available on this machine.""" + log_raw("=== SIMULATOR RUNTIME VALIDATION ===", tee=True) + result = run_cmd(["xcrun", "simctl", "list", "runtimes"], check=False) + if result.returncode != 0: + log("WARNING: 'xcrun simctl list runtimes' failed. " + "Simulator may not work.", tee=True) + return + + # Check that at least one iOS runtime is listed + ios_runtimes = [line for line in (result.stdout or "").splitlines() + if "iOS" in line] + if ios_runtimes: + log(f"Found {len(ios_runtimes)} iOS runtime(s):", tee=True) + for rt in ios_runtimes: + log(f" {rt.strip()}") + else: + log("WARNING: No iOS simulator runtimes found. " + "Simulator-based testing will fail.", tee=True) + + +# Preferred iPhone device types in priority order. Used to pick a device type +# when creating our own simulator. We prefer plain models over Pro/Pro Max for +# predictable performance characteristics, and skip mini/SE variants because +# their availability across Xcode versions is inconsistent. The list is wide +# enough to survive Xcode/runtime upgrades on Helix machines. +_PREFERRED_IPHONE_MODELS = [ + "iPhone 17", "iPhone 17 Pro", "iPhone 17 Pro Max", "iPhone Air", + "iPhone 16", "iPhone 16 Pro", "iPhone 16 Pro Max", "iPhone 16 Plus", + "iPhone 15", "iPhone 15 Pro", "iPhone 15 Pro Max", "iPhone 15 Plus", + "iPhone 14", "iPhone 14 Pro", "iPhone 14 Pro Max", "iPhone 14 Plus", + "iPhone 13", "iPhone 13 Pro", "iPhone 13 Pro Max", + "iPhone 12", "iPhone 12 Pro", "iPhone 12 Pro Max", + "iPhone 11", "iPhone 11 Pro", "iPhone 11 Pro Max", +] + + +def _find_ios_runtime(): + """Return the (identifier, version) of the highest-version available iOS runtime. + + Returns ``(None, None)`` if no iOS runtime is available. + """ + result = run_cmd(["xcrun", "simctl", "list", "runtimes", "-j"], check=False) + if result.returncode != 0 or not result.stdout: + return (None, None) + + try: + data = json.loads(result.stdout) + except json.JSONDecodeError as e: + log(f"WARNING: Failed to parse runtimes JSON: {e}") + return (None, None) + + available = [] + for rt in data.get("runtimes", []): + if not rt.get("isAvailable"): + continue + name = rt.get("name", "") + if "iOS" not in name: + continue + identifier = rt.get("identifier", "") + version = rt.get("version", "0") + available.append((identifier, version, name)) + + if not available: + return (None, None) + + # Sort by version (descending) so the latest iOS runtime wins. version + # strings like "26.4.1" sort correctly with packaging-style key, but we + # only need a stable preference for the latest, so simple tuple sort works. + def _version_key(item): + try: + return tuple(int(p) for p in item[1].split(".") if p.isdigit()) + except Exception: + return (0,) + + available.sort(key=_version_key, reverse=True) + rt_id, rt_ver, rt_name = available[0] + log(f"Selected iOS runtime: {rt_name} (id={rt_id}, version={rt_ver})", tee=True) + return (rt_id, rt_ver) + + +def _find_iphone_device_type(): + """Return the identifier of a usable iPhone simulator device type. + + Walks ``_PREFERRED_IPHONE_MODELS`` in order and returns the identifier of + the first model present in ``simctl list devicetypes -j``. Returns ``None`` + if none of the preferred models are available. + """ + result = run_cmd(["xcrun", "simctl", "list", "devicetypes", "-j"], check=False) + if result.returncode != 0 or not result.stdout: + return None + + try: + data = json.loads(result.stdout) + except json.JSONDecodeError as e: + log(f"WARNING: Failed to parse devicetypes JSON: {e}") + return None + + by_name = {dt.get("name", ""): dt.get("identifier", "") + for dt in data.get("devicetypes", [])} + for model in _PREFERRED_IPHONE_MODELS: + if model in by_name and by_name[model]: + log(f"Selected iPhone device type: {model} (id={by_name[model]})", tee=True) + return by_name[model] + + log(f"WARNING: None of the preferred iPhone models are installed. " + f"Available iPhones: {[n for n in by_name if n.startswith('iPhone')]}", + tee=True) + return None + + +def _unique_simulator_name(): + """Build a per-workitem unique simulator name to avoid collisions. + + Each Helix work item gets its own name so concurrent work items on the + same machine never share or delete each other's simulators. + """ + workitem_id = (os.environ.get("HELIX_WORKITEM_ID") or + os.environ.get("HELIX_WORKITEM_FRIENDLYNAME") or + "") + if workitem_id: + # Strip filesystem-unfriendly chars from friendly names like + # "Inner Loop Simulator - MAUI iOS Inner Loop" + suffix = re.sub(r"[^A-Za-z0-9_-]+", "-", workitem_id).strip("-")[:48] + else: + # Fallback: epoch + PID. Still unique within a single machine session. + suffix = f"{int(datetime.now().timestamp())}-{os.getpid()}" + return f"PerfTest-iPhone-{suffix}" + + +def _write_sim_udid(workitem_root, udid): + """Persist the UDID of the simulator we created so test.py / post.py can + read it. Without this, ioshelper would fall back to "first booted device" + which is unsafe if any other simulator happens to be booted. + """ + path = os.path.join(workitem_root, "sim_udid.txt") + try: + with open(path, "w", encoding="utf-8") as f: + f.write(udid + "\n") + log(f"Wrote simulator UDID to {path}", tee=True) + except OSError as e: + log(f"WARNING: Could not write {path}: {e}", tee=True) + + +def _simulator_preflight(udid): + """Verify the booted simulator is actually usable for spawning processes. + + The CoreSimulator daemon can list a device as "Booted" while still + refusing to spawn agents (for example when the device's underlying data + container is owned by a different user — the failure mode that breaks + actool's AssetCatalogSimulatorAgent during ``dotnet build``). + Running ``simctl spawn /usr/bin/true`` proves end-to-end that we + can launch processes inside the simulator under the current user. + Raises ``SystemExit(1)`` if the spawn fails. + """ + log(f"Preflight: spawning /usr/bin/true inside simulator {udid}", tee=True) + result = run_cmd( + ["xcrun", "simctl", "spawn", udid, "/usr/bin/true"], + check=False, + ) + if result.returncode != 0: + log(f"ERROR: simctl spawn preflight failed (exit {result.returncode}). " + f"CoreSimulator cannot launch processes in this device. " + f"actool / mlaunch will also fail.", tee=True) + _dump_log() + sys.exit(1) + log("Preflight OK — simulator can spawn processes.", tee=True) + + +def _sweep_leaked_perftest_simulators(): + """Shutdown and delete any leaked ``PerfTest-iPhone-*`` simulators from + previous workitems on this Helix machine. + + Each booted simulator forks ~150–200 child processes (launchd_sim plus + a long tail of system daemons). A handful of leaked simulators is enough + to push the per-user process count past the macOS ``maxUserProcs`` + rlimit (typically 1333), at which point ``simctl boot`` for THIS work + item fails with NSPOSIXErrorDomain code 67 / "Unable to boot device + due to insufficient system resources". Leaks happen when a previous + workitem's post.py crashed before reaching delete_simulator(), or when + the workitem was killed mid-run by a Helix timeout. + + We're conservative: we only touch devices whose name starts with the + well-known ``PerfTest-iPhone-`` prefix used by ``_unique_simulator_name``, + so we never disturb shared queue infrastructure or non-perf workitems. + Best-effort — failures are logged but don't abort setup. + """ + log_raw("=== LEAKED PERFTEST SIMULATOR SWEEP ===", tee=True) + try: + listing = run_cmd(["xcrun", "simctl", "list", "devices", "-j"], + check=False) + except Exception as e: + log(f"WARNING: could not list simulators for sweep: {e}", tee=True) + return + if listing.returncode != 0 or not listing.stdout: + log(f"WARNING: simctl list devices -j failed (exit {listing.returncode}); " + "skipping sweep.", tee=True) + return + try: + devices_by_runtime = json.loads(listing.stdout).get("devices", {}) + except (json.JSONDecodeError, AttributeError) as e: + log(f"WARNING: could not parse simctl device JSON for sweep: {e}", + tee=True) + return + + leaked = [] + for runtime_devices in devices_by_runtime.values(): + for dev in runtime_devices or []: + name = (dev.get("name") or "").strip() + udid = (dev.get("udid") or "").strip() + if name.startswith("PerfTest-iPhone-") and udid: + leaked.append((name, udid, dev.get("state", "Unknown"))) + + if not leaked: + log("No leaked PerfTest-iPhone-* simulators found.", tee=True) + return + + log(f"Found {len(leaked)} leaked PerfTest-iPhone-* simulator(s); " + "shutting down and deleting...", tee=True) + for name, udid, state in leaked: + log(f" Cleaning {name} ({udid}, state={state})", tee=True) + # shutdown is best-effort and idempotent on already-shutdown devices + run_cmd(["xcrun", "simctl", "shutdown", udid], check=False) + run_cmd(["xcrun", "simctl", "delete", udid], check=False) + + +def create_and_boot_simulator(workitem_root): + """Create a fresh iPhone simulator under the current user's CoreSimulator + namespace and boot it. + + Why a fresh device every run: + Helix machines accumulate simulator devices across runs. When those + pre-existing devices are owned by a different user (root, or a previous + tenant), ``simctl boot`` on them fails with NSCocoaErrorDomain code 513 + ("You don't have permission to save the file ... in the folder + CoreSimulator"). Creating our own device with ``simctl create`` writes + to ``~/Library/Developer/CoreSimulator/Devices/`` of THIS user, so we + always have a writable device to boot. + + Why both simulator AND device jobs need this: + ``dotnet build`` for ios-arm64 invokes ``actool`` which spawns + ``AssetCatalogSimulatorAgent`` via CoreSimulator to compile the asset + catalog. Without a writable, booted simulator under our user, that + spawn fails and the build aborts. + + Returns the UDID of the booted device. Persists it to + ``/sim_udid.txt`` for test.py / post.py to read. + Exits with code 1 on any unrecoverable failure. + """ + log_raw("=== SIMULATOR CREATE & BOOT ===", tee=True) + + # Defensive cleanup: kill any leaked PerfTest-iPhone-* simulators from + # previous workitems before we try to boot our own. Without this, on a + # machine where prior post.py runs crashed or were killed by timeout, + # the per-user process count stays pinned just below maxUserProcs and + # this workitem's simctl boot fails with exit 67. Only touches devices + # we own (PerfTest-iPhone-* naming) so other workitems are unaffected. + _sweep_leaked_perftest_simulators() + + runtime_id, _ = _find_ios_runtime() + if not runtime_id: + log("ERROR: No available iOS simulator runtime found.", tee=True) + run_cmd(["xcrun", "simctl", "list", "runtimes"], check=False) + _dump_log() + sys.exit(1) + + device_type_id = _find_iphone_device_type() + if not device_type_id: + log("ERROR: No usable iPhone simulator device type found.", tee=True) + run_cmd(["xcrun", "simctl", "list", "devicetypes"], check=False) + _dump_log() + sys.exit(1) + + name = _unique_simulator_name() + log(f"Creating simulator: name='{name}' type={device_type_id} runtime={runtime_id}", + tee=True) + create = run_cmd( + ["xcrun", "simctl", "create", name, device_type_id, runtime_id], + check=False, + ) + if create.returncode != 0 or not create.stdout: + log(f"ERROR: simctl create failed (exit {create.returncode}).", tee=True) + _dump_log() + sys.exit(1) + + # `simctl create` prints just the UDID on stdout (one line). + udid = (create.stdout or "").strip().splitlines()[-1].strip() + if not re.match(r"^[0-9A-Fa-f-]{36}$", udid): + log(f"ERROR: simctl create produced unexpected output: {create.stdout!r}", + tee=True) + _dump_log() + sys.exit(1) + log(f"Created simulator UDID: {udid}", tee=True) + + log(f"Booting simulator {udid}", tee=True) + boot = run_cmd(["xcrun", "simctl", "boot", udid], check=False) + if boot.returncode != 0 and "Booted" not in (boot.stdout or ""): + # On shared Helix machines, even a fresh device can fail to boot + # because CoreSimulator writes log/state files into shared folders + # that may still have foreign ownership the chown didn't catch. + # Run another chown then retry once. If the retry still fails, give + # up — the diagnostics from chown should explain why. + log(f"WARNING: simctl boot failed (exit {boot.returncode}). " + f"Re-running CoreSimulator chown and retrying once.", tee=True) + fix_coresimulator_permissions() + boot = run_cmd(["xcrun", "simctl", "boot", udid], check=False) + if boot.returncode != 0 and "Booted" not in (boot.stdout or ""): + log(f"ERROR: simctl boot retry failed (exit {boot.returncode}).", + tee=True) + _dump_log() + sys.exit(1) + log("Boot retry succeeded after chown.", tee=True) + + # Wait for the simulator to finish booting. Without this, downstream + # actool / mlaunch race against the boot and intermittently fail. + log(f"Waiting for boot to complete (simctl bootstatus -b)", tee=True) + bootstatus = run_cmd(["xcrun", "simctl", "bootstatus", udid, "-b"], check=False) + if bootstatus.returncode != 0: + log(f"WARNING: bootstatus reported exit {bootstatus.returncode}; " + f"continuing anyway and relying on preflight.", tee=True) + + _simulator_preflight(udid) + _write_sim_udid(workitem_root, udid) + + log("Currently booted devices:") + run_cmd(["xcrun", "simctl", "list", "devices", "booted"], check=False) + return udid + + +def _run_workload_cmd(args, timeout_seconds): + """Run a workload install command with a timeout. + + Returns a CompletedProcess. If the command times out, kills the process + and returns a synthetic CompletedProcess with returncode=-1. + + The dotnet CLI can hang for hours when NuGet feeds are slow or broken + (internal download retries). A timeout ensures the fallback retry has + time to run within the Helix work item timeout. + """ + try: + return run_cmd(args, check=False, timeout=timeout_seconds) + except subprocess.TimeoutExpired as e: + log(f"WARNING: Command timed out after {timeout_seconds}s — killed", tee=True) + # Log tail of any partial output captured before the kill + partial = getattr(e, 'output', '') or '' + if partial: + if not isinstance(partial, str): + partial = partial.decode('utf-8', errors='replace') + for line in partial.splitlines()[-20:]: + log_raw(line) + return subprocess.CompletedProcess(args, returncode=-1) + + +# Cap each workload install attempt so there's time for the fallback retry +# within the Helix work item timeout (2:30). Without this, the dotnet CLI +# can hang for 2+ hours on NuGet download failures (internal retries). +_WORKLOAD_INSTALL_TIMEOUT = 1200 # 20 minutes per attempt + +# NuGet restore should complete much faster than workload install, but can +# still hang on dead feeds. 10 minutes is generous; prevents consuming the +# entire Helix work item timeout (2:30) on a hung restore. +_RESTORE_TIMEOUT = 600 # 10 minutes + + +def install_workload(ctx): + """Install the maui-ios workload using the shipped SDK. + + Uses the rollback file created by pre.py to pin to the exact workload + version (latest nightly packs). Falls back to a plain install if no + rollback file is present. Always uses --ignore-failed-sources because + dead NuGet feeds are common in CI. + + Each attempt is capped at _WORKLOAD_INSTALL_TIMEOUT seconds to prevent + slow NuGet downloads from consuming the entire Helix work item timeout. + + Manifest-patching dependency (pre.py): + When the iOS workload manifest references net10.0 cross-targeting + packs that don't exist on NuGet, pre.py patches the manifest to + remove those entries and places the patched files inside the SDK tree + at ``$DOTNET_ROOT/sdk-manifests/{band}/microsoft.net.sdk.ios/{ver}/``. + The SDK tree ships to Helix as the correlation payload, so the + patched manifest is already on disk. The ``--skip-manifest-update`` + retry below tells the CLI to use on-disk manifests instead of + downloading new ones, which picks up the patched manifest and only + installs packs that actually exist. + """ + log_raw("=== WORKLOAD INSTALL ===", tee=True) + + rollback_file = os.path.join(ctx["workitem_root"], "rollback_maui.json") + nuget_config = ctx["nuget_config"] + + install_args = [ + ctx["dotnet_exe"], "workload", "install", "maui-ios", + ] + + if os.path.isfile(rollback_file): + log(f"Using rollback file: {rollback_file}") + install_args.extend(["--from-rollback-file", rollback_file]) + else: + log("No rollback_maui.json found — installing latest maui-ios workload") + + if os.path.isfile(nuget_config): + install_args.extend(["--configfile", nuget_config]) + + # Dead NuGet feeds are common in CI — always tolerate failures + install_args.append("--ignore-failed-sources") + + result = _run_workload_cmd(install_args, _WORKLOAD_INSTALL_TIMEOUT) + if result.returncode != 0 and os.path.isfile(rollback_file): + # The --from-rollback-file attempt typically fails for one of two + # reasons: + # 1. NuGet version skew — packs referenced by a new manifest have + # not propagated to all NuGet feeds yet ("package NOT FOUND"). + # 2. net10.0 cross-targeting packs — the manifest references packs + # that don't exist on any feed (upstream coherency issue). + # + # In case (2), pre.py's manifest-patching fallback has already + # placed a patched manifest (with net10.0 entries removed) inside + # the SDK tree, which was shipped here as the correlation payload. + # --skip-manifest-update tells the CLI to use that on-disk manifest + # instead of downloading a new one, so it only resolves packs that + # actually exist. + log(f"WARNING: Workload install with rollback file failed " + f"(exit code {result.returncode})", tee=True) + log("Retrying with --skip-manifest-update (uses on-disk manifest " + "from correlation payload)...", tee=True) + + retry_args = [ + ctx["dotnet_exe"], "workload", "install", "maui-ios", + "--skip-manifest-update", + ] + if os.path.isfile(nuget_config): + retry_args.extend(["--configfile", nuget_config]) + retry_args.append("--ignore-failed-sources") + + result = _run_workload_cmd(retry_args, _WORKLOAD_INSTALL_TIMEOUT) + + if result.returncode != 0: + log(f"WORKLOAD INSTALL FAILED (exit code {result.returncode})", tee=True) + _dump_log() + sys.exit(1) + + log("maui-ios workload installed successfully") + + +def restore_packages(ctx): + """Restore NuGet packages for the app project. + + Uses --ignore-failed-sources and /p:NuGetAudit=false to handle dead + feeds and avoid audit warnings that slow down restore. + """ + log_raw("=== NUGET RESTORE ===", tee=True) + + csproj = ctx["csproj"] + if not os.path.isfile(csproj): + log(f"ERROR: Project file not found at {csproj}", tee=True) + _dump_log() + sys.exit(2) + + restore_args = [ + ctx["dotnet_exe"], "restore", csproj, + "--ignore-failed-sources", + "/p:NuGetAudit=false", + ] + + nuget_config = ctx["nuget_config"] + if os.path.isfile(nuget_config): + restore_args.extend(["--configfile", nuget_config]) + + framework = ctx.get("framework") + if framework: + restore_args.append(f"/p:TargetFrameworks={framework}") + + msbuild_args = ctx.get("msbuild_args") + if msbuild_args: + restore_args.extend(msbuild_args.split()) + + try: + result = run_cmd(restore_args, check=False, timeout=_RESTORE_TIMEOUT) + except subprocess.TimeoutExpired: + log(f"RESTORE TIMED OUT after {_RESTORE_TIMEOUT}s", tee=True) + _dump_log() + sys.exit(2) + if result.returncode != 0: + log(f"RESTORE FAILED (exit code {result.returncode})", tee=True) + _dump_log() + sys.exit(2) + + log("NuGet restore succeeded") + + +def disable_spotlight(workitem_root): + """Disable Spotlight indexing on the workitem directory. + + Spotlight's mds_stores process can hold file locks during builds, + causing intermittent build failures. This is a well-known issue on + macOS CI machines. + """ + log_raw("=== DISABLE SPOTLIGHT ===", tee=True) + result = run_cmd( + ["sudo", "mdutil", "-i", "off", workitem_root], + check=False, + ) + if result.returncode != 0: + # Non-fatal — Spotlight interference is intermittent + log(f"WARNING: mdutil -i off failed (exit {result.returncode}). " + "Spotlight may interfere with builds.", tee=True) + else: + log(f"Spotlight indexing disabled for {workitem_root}") + + +def detect_physical_device(): + """Detect whether a physical iOS device is connected and return its UDID. + + Checks IOS_DEVICE_UDID env var first, then uses 'xcrun devicectl list devices'. + Returns the UDID string, or None if no device is found. + + NOTE: This duplicates iOSHelper.detect_connected_device() intentionally. + setup_helix.py runs as a standalone Helix pre-command with minimal imports + (no performance.common, no shared.ioshelper). Keeping this self-contained + avoids import failures on the Helix machine. + """ + log_raw("=== PHYSICAL DEVICE DETECTION ===", tee=True) + + udid = os.environ.get("IOS_DEVICE_UDID", "").strip() + if udid: + log(f"Using IOS_DEVICE_UDID from environment: {udid}", tee=True) + return udid + + # Auto-detect via devicectl + result = run_cmd( + ["xcrun", "devicectl", "list", "devices"], + check=False, + ) + if result.returncode != 0: + log("WARNING: 'xcrun devicectl list devices' failed. " + "No physical device detection available.", tee=True) + return None + + # Log the full output for debugging + log("devicectl output:") + for line in (result.stdout or "").splitlines(): + log_raw(f" {line}") + + # Try JSON output for structured parsing + # Write to temp file instead of /dev/stdout because devicectl mixes + # human-readable table text and JSON when writing to stdout. + import tempfile + fd, json_tmp = tempfile.mkstemp(suffix='.json', prefix='devicectl_') + os.close(fd) + try: + json_result = run_cmd( + ["xcrun", "devicectl", "list", "devices", "--json-output", json_tmp], + check=False, + ) + if json_result.returncode == 0 and os.path.exists(json_tmp): + try: + import json + with open(json_tmp, 'r') as f: + data = json.load(f) + devices = data.get("result", {}).get("devices", []) + for device in devices: + conn = device.get("connectionProperties", {}) + transport = conn.get("transportType", "") + name = device.get("deviceProperties", {}).get("name", "unknown") + # Prefer hardware UDID (e.g. 00008020-001965D83C43002E) over + # CoreDevice identifier (a UUID). mlaunch requires the hardware + # UDID — same logic as iOSHelper.detect_connected_device(). + hw_udid = device.get("hardwareProperties", {}).get("udid", "") + device_udid = hw_udid or device.get("identifier", "") + if transport in ("wired", "localNetwork", "wifi") and device_udid: + log(f"Found connected device: {name} (UDID: {device_udid}, " + f"transport: {transport})", tee=True) + return device_udid + except Exception as e: + log(f"JSON parsing failed: {e}", tee=True) + finally: + if os.path.exists(json_tmp): + os.remove(json_tmp) + + log("No connected physical devices found.", tee=True) + return None + + +# --- Main --- + +# --- Device signing artifact discovery --- + +# On Mac.iPhone.*.Perf machines, Helix machine prep installs the developer +# provisioning profile (embedded.mobileprovision) and the 'sign' tool at +# known paths. XHarness work items get them in CWD automatically; HelixWorkItem +# (us) has to find and stage them ourselves. +_SIGNING_SEARCH_ROOTS = [ + "/etc/helix-prep", + "/Users/helix-runner", + "/Users/Shared/Helix", + "/var/helix", + "/usr/local/bin", + "/usr/local/share", +] + + +def find_and_stage_signing_artifacts(workitem_root): + """Locate embedded.mobileprovision and the 'sign' tool on the Helix machine. + + Searches several well-known root directories. If found, stages + embedded.mobileprovision into ``workitem_root`` (so ioshelper.py picks + it up via its CWD-based lookup) and symlinks ``sign`` into the work + item's venv ``bin/`` directory (already on PATH per the .proj + PreCommands), so ioshelper.py finds it via ``shutil.which('sign')``. + + Returns True if BOTH artifacts were found and staged, False otherwise. + Does not raise; the caller decides what to do with the result. + """ + log_raw("=== DEVICE SIGNING ARTIFACT DISCOVERY ===", tee=True) + + provision_path = None + sign_path = None + for root in _SIGNING_SEARCH_ROOTS: + if not os.path.isdir(root): + continue + try: + result = subprocess.run( + [ + "find", root, "-maxdepth", "6", + "(", "-name", "embedded.mobileprovision", + "-o", "-name", "sign", ")", + "-not", "-path", "*/.Trash/*", + ], + capture_output=True, text=True, timeout=30, + ) + for line in (result.stdout or "").splitlines(): + line = line.strip() + if not line: + continue + base = os.path.basename(line) + if base == "embedded.mobileprovision" and provision_path is None: + provision_path = line + elif base == "sign" and sign_path is None: + if os.path.isfile(line) and os.access(line, os.X_OK): + sign_path = line + if provision_path and sign_path: + break + except Exception as e: + log(f"Search in {root} failed: {e}") + if provision_path and sign_path: + break + + if provision_path: + log(f"Found embedded.mobileprovision at: {provision_path}", tee=True) + try: + import shutil + dest = os.path.join(workitem_root, "embedded.mobileprovision") + shutil.copy2(provision_path, dest) + log(f"Copied embedded.mobileprovision to: {dest}", tee=True) + except Exception as e: + log(f"WARNING: failed to copy embedded.mobileprovision: {e}", tee=True) + provision_path = None + else: + log("WARNING: embedded.mobileprovision not found in any known location. " + "Device install will likely fail with 'No code signature found'.", + tee=True) + + if sign_path: + log(f"Found 'sign' tool at: {sign_path}", tee=True) + # Symlink into the workitem venv bin (already on PATH per .proj PreCommands) + venv_bin = os.path.join(workitem_root, ".venv", "bin") + try: + os.makedirs(venv_bin, exist_ok=True) + link_target = os.path.join(venv_bin, "sign") + if os.path.lexists(link_target): + os.remove(link_target) + os.symlink(sign_path, link_target) + log(f"Symlinked sign tool to: {link_target}", tee=True) + except Exception as e: + log(f"WARNING: failed to symlink sign tool: {e}", tee=True) + sign_path = None + else: + log("WARNING: 'sign' tool not found in any known location. " + "Device install will likely fail. Searched: " + f"{', '.join(_SIGNING_SEARCH_ROOTS)}", tee=True) + + return bool(provision_path and sign_path) + + +def main(): + global _logfile + + # Open log file in HELIX_WORKITEM_UPLOAD_ROOT for post-mortem debugging + upload_root = os.environ.get("HELIX_WORKITEM_UPLOAD_ROOT") + if upload_root: + os.makedirs(upload_root, exist_ok=True) + _logfile = open(os.path.join(upload_root, "output.log"), "a") + + workitem_root = os.environ.get("HELIX_WORKITEM_ROOT", ".") + correlation_payload = os.environ.get("HELIX_CORRELATION_PAYLOAD", ".") + + # Determine target device type from iOSRid env var (set by .proj). + # ios-arm64 → physical device, iossimulator-* → simulator + ios_rid = os.environ.get("IOS_RID", "iossimulator-x64") + is_physical_device = (ios_rid == "ios-arm64") + + # Detect host architecture to select the correct simulator RID. + # Mac.iPhone.17.Perf queue uses Intel x64 machines which need + # iossimulator-x64, not iossimulator-arm64. Apple Silicon needs + # iossimulator-arm64. Physical device builds (ios-arm64) target the + # iPhone hardware, not the Mac, so skip architecture override. + if not is_physical_device: + host_arch = platform.machine() + if host_arch == "x86_64": + ios_rid = "iossimulator-x64" + elif host_arch == "arm64": + ios_rid = "iossimulator-arm64" + else: + log(f"WARNING: Unknown architecture '{host_arch}', " + f"keeping IOS_RID={ios_rid}", tee=True) + os.environ["IOS_RID"] = ios_rid + log(f"Host architecture: {host_arch}, using IOS_RID={ios_rid}", tee=True) + + # The simulator device name env var is no longer used: we always create + # our own fresh simulator under this user's CoreSimulator namespace via + # create_and_boot_simulator(). Pre-existing devices on Helix machines + # may belong to other users and refuse to boot with permission errors. + + # Framework and MSBuild args are passed as command-line arguments when + # available (from the .proj PreCommands), or fall back to env vars. + framework = sys.argv[1] if len(sys.argv) > 1 else os.environ.get("PERFLAB_Framework", "") + msbuild_args = sys.argv[2] if len(sys.argv) > 2 else "" + + ctx = { + "framework": framework, + "msbuild_args": msbuild_args, + "workitem_root": workitem_root, + "correlation_payload": correlation_payload, + "nuget_config": os.path.join(workitem_root, "app", "NuGet.config"), + "csproj": os.path.join(workitem_root, "app", "MauiiOSInnerLoop.csproj"), + } + + log_raw("=== iOS HELIX SETUP START ===", tee=True) + log(f"Target device type: {'physical device' if is_physical_device else 'simulator'} " + f"(IOS_RID={ios_rid})", tee=True) + + # Step 1: Configure the .NET SDK from the correlation payload + ctx["dotnet_exe"] = setup_dotnet(correlation_payload) + print_diagnostics() + + # Step 2: Select the Xcode version matching the iOS SDK workload packs. + # The packs require a specific Xcode (e.g. 26.2.x packs need Xcode 26.2). + # Selecting early avoids wasting 20+ min on workload install before + # _ValidateXcodeVersion fails. + select_xcode(workitem_root) + + # Step 3 & 4: Device-type-specific setup. + # Both job types create + boot a fresh simulator under THIS user's + # CoreSimulator namespace. Pre-existing simulators on the Helix machine + # may belong to other users (root or previous tenants) and refuse to boot + # with permission errors. Even physical-device builds need a writable + # simulator booted because actool spawns AssetCatalogSimulatorAgent via + # CoreSimulator during 'dotnet build' for ios-arm64. + fix_coresimulator_permissions() + validate_simulator_runtimes() + create_and_boot_simulator(workitem_root) + + if is_physical_device: + # Order matters: detect the physical device FIRST, signing artifacts + # SECOND. Both gates check independent infra (one is hardware on the + # USB hub, the other is files in the keychain / on disk), and either + # can fail by itself. Detecting the device first means the log always + # shows what hardware the queue exposed regardless of signing state, + # which is what humans actually need to debug a red work item. + + # Detect and validate the connected physical device. We require one; + # there is NO fallback to the simulator on a device job. A green + # device job MUST mean device measurements ran on real hardware, + # never that we silently downgraded to a simulator. + device_udid = detect_physical_device() + if not device_udid: + reason = ( + "No physical iOS device detected on this Helix machine. " + "Device job requires a connected, paired iPhone — there is " + "NO simulator fallback by design. This is a queue " + "provisioning gap (device disconnected, not paired, or " + "queue capacity issue), not a scenario bug. Verify the " + "Mac.iPhone.13.Perf machine has its iPhone connected and " + "paired (xcrun devicectl list devices)." + ) + log_raw("=" * 70, tee=True) + log_raw("WORK ITEM FAILED — NO PHYSICAL DEVICE", tee=True) + log_raw("=" * 70, tee=True) + log(reason, tee=True) + log_raw("=" * 70, tee=True) + _dump_log() + sys.exit(1) + + # Log the detected UDID for diagnostics. Note: os.environ changes + # in this Python process do NOT persist to subsequent Helix commands + # (test.py, post.py). runner.py re-detects the device independently + # via iOSHelper.detect_connected_device(). + os.environ["IOS_DEVICE_UDID"] = device_udid + log(f"IOS_DEVICE_UDID detected: {device_udid}", tee=True) + + # Search for embedded.mobileprovision and 'sign' tool on the + # Helix machine and stage them so ioshelper.py's signing flow + # can find them. + signing_ready = find_and_stage_signing_artifacts(workitem_root) + + # Without code-signing infrastructure (cert in keychain + + # provisioning profile + 'sign' tool), iOS device install + # cannot succeed — devicectl will fail with + # "No code signature found" regardless of which install tool + # we use. Fail loudly so missing queue provisioning shows up + # as a red build, not a green-with-hidden-skip. Provisioning + # the queue (Apple Developer cert + provisioning profile + + # 'sign' tool, same as Mac.iPhone.17.Perf) is an Engineering + # Services ticket, not a code change here. + if not signing_ready: + reason = ( + "Device code-signing infrastructure not available on this " + "Helix machine (embedded.mobileprovision and/or 'sign' tool " + "missing). Cannot install signed app on physical device. " + "This is a queue provisioning gap, not a scenario bug. " + "Fix: provision the queue with the Apple Developer cert + " + "provisioning profile + 'sign' tool (same as " + "Mac.iPhone.17.Perf). Search roots checked: " + + ", ".join(_SIGNING_SEARCH_ROOTS) + ) + log_raw("=" * 70, tee=True) + log_raw("WORK ITEM FAILED — DEVICE INFRA UNAVAILABLE", tee=True) + log_raw("=" * 70, tee=True) + log(reason, tee=True) + log_raw("=" * 70, tee=True) + _dump_log() + sys.exit(1) + log(f"IOS_DEVICE_UDID detected: {device_udid}", tee=True) + + # Step 5: Install the maui-ios workload + # Must happen BEFORE restore because restore needs workload packs + install_workload(ctx) + + # Step 6: Restore NuGet packages + restore_packages(ctx) + + # Step 7: Disable Spotlight indexing to prevent file-lock errors + disable_spotlight(workitem_root) + + log_raw("=== iOS HELIX SETUP SUCCEEDED ===", tee=True) + _dump_log() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/scenarios/mauiiosinnerloop/test.py b/src/scenarios/mauiiosinnerloop/test.py new file mode 100644 index 00000000000..a19a6cfabdb --- /dev/null +++ b/src/scenarios/mauiiosinnerloop/test.py @@ -0,0 +1,14 @@ +''' +MAUI iOS Inner Loop (Debug End-to-End) Time Measurement +Orchestrates first build-deploy-startup → file edit → incremental build-deploy-startup → parse binlogs and startup times. +''' +from shared.runner import TestTraits, Runner + +EXENAME = 'MauiiOSInnerLoop' + + +if __name__ == "__main__": + traits = TestTraits(exename=EXENAME, + guiapp='false', + ) + Runner(traits).run() diff --git a/src/scenarios/shared/const.py b/src/scenarios/shared/const.py index 074fa7fdf89..c1dc75f7d91 100644 --- a/src/scenarios/shared/const.py +++ b/src/scenarios/shared/const.py @@ -19,6 +19,7 @@ ANDROIDINSTRUMENTATION = "androidinstrumentation" DEVICEPOWERCONSUMPTION = "devicepowerconsumption" BUILDTIME = "buildtime" +IOSINNERLOOP = "iosinnerloop" SCENARIO_NAMES = {STARTUP: 'Startup', SDK: 'SDK', @@ -27,7 +28,8 @@ INNERLOOP: 'Innerloop', INNERLOOPMSBUILD: 'InnerLoopMsBuild', DOTNETWATCH: 'DotnetWatch', - BUILDTIME: 'BuildTime'} + BUILDTIME: 'BuildTime', + IOSINNERLOOP: 'IOSInnerLoop'} BINDIR = 'bin' PUBDIR = 'pub' diff --git a/src/scenarios/shared/ioshelper.py b/src/scenarios/shared/ioshelper.py new file mode 100644 index 00000000000..6f9ede30c57 --- /dev/null +++ b/src/scenarios/shared/ioshelper.py @@ -0,0 +1,729 @@ +"""iOS deploy and startup measurement helpers (simulator + physical device). + +This module is the single source of truth for "F5-style" install + launch +measurement on iOS. Two device kinds, two slightly different toolchains: + + ┌──────────┬──────────────────────────┬──────────────────────────────┐ + │ │ install │ cold-startup measurement │ + ├──────────┼──────────────────────────┼──────────────────────────────┤ + │ device │ mlaunch --installdev │ mlaunch --launchdev │ + │ │ (handles devicectl tunnel│ (returns PID immediately, │ + │ │ + devicectl signing │ watchdog log captures the │ + │ │ metadata) │ startup phases) │ + ├──────────┼──────────────────────────┼──────────────────────────────┤ + │ simulator│ mlaunch --installsim │ xcrun simctl launch │ + │ │ (matches what the IDE │ (NOT mlaunch --launchsim, │ + │ │ does during F5) │ see Note 1 below) │ + └──────────┴──────────────────────────┴──────────────────────────────┘ + +Per .NET iOS team guidance (Rolf Bjarne Kvinge), mlaunch is the canonical +tool the IDEs use during F5 — it handles ad-hoc signing, devicectl tunnel +setup, and stdout tunnelling. We use it wherever the IDE would. + +Note 1 — why simctl for simulator launch instead of mlaunch --launchsim: + mlaunch --launchsim blocks until the launched app exits (it tunnels app + stdout/stderr) and during testing on Apple Silicon Helix machines the + simulator transitioned from Booted → Shutdown silently during the call, + producing 180-second timeouts with no diagnostic output. simctl launch + is what mlaunch invokes internally for the actual launch step, so the + wall-clock measurement is equivalent — we just skip mlaunch's stdout + tunnel layer (we don't need it for measurement). See + ``measure_cold_startup`` for the implementation and the post-launch + stabilization check that confirms the launched PID survives. + +Note 2 — physical-device install requires code signing: + ``mlaunch --installdev`` ultimately calls ``xcrun devicectl device + install app``, which refuses to install an unsigned bundle (errors with + MIInstallerErrorDomain code 13, "No code signature found"). The .proj + builds with ``EnableCodeSigning=false`` to keep the build deterministic + on Helix, then ``sign_app_for_device`` re-signs the bundle using the + ``embedded.mobileprovision`` and ``sign`` tool that Helix machine prep + is expected to provide. If those artifacts are missing the install + will fail; the orchestrator (setup_helix.py) is responsible for + detecting that case. +""" + +import glob +import json +import os +import re +import subprocess +import tempfile +import time +from datetime import datetime +from logging import getLogger + +from performance.common import RunCommand + + +class iOSHelper: + """Unified helper for iOS simulator and physical device operations. + + Callers use the same API (setup_device, install_app, measure_cold_startup, + cleanup) regardless of device type. See module docstring for the + install/launch tooling matrix and the rationale behind it. + """ + + _mlaunch_path = None # resolved once, cached for the process + + def __init__(self): + self.bundle_id = None + self.device_id = None + self.app_bundle_path = None + self.is_physical_device = False + + # ── mlaunch Resolution ──────────────────────────────────────────── + + @staticmethod + def _resolve_mlaunch(): + """Resolve the mlaunch binary from the iOS SDK pack. + + Searches $DOTNET_ROOT/packs/Microsoft.iOS.Sdk.*/tools/bin/mlaunch, + falling back to ~/.dotnet if DOTNET_ROOT is unset. Caches the result. + """ + if iOSHelper._mlaunch_path is not None: + return iOSHelper._mlaunch_path + + dotnet_root = os.environ.get('DOTNET_ROOT', os.path.expanduser('~/.dotnet')) + pattern = os.path.join(dotnet_root, 'packs', 'Microsoft.iOS.Sdk.*', '*', 'tools', 'bin', 'mlaunch') + matches = glob.glob(pattern) + if not matches: + raise FileNotFoundError( + f"mlaunch not found. Searched: {pattern}\n" + f"Ensure the iOS SDK workload is installed (dotnet workload install ios)." + ) + + def _version_key(p: str): + m = re.search(r'Microsoft\.iOS\.Sdk\.([^/\\]+)', p) + if not m: + return () + parts = re.split(r'[.\-+]', m.group(1)) + key = [] + for part in parts: + key.append((0, int(part)) if part.isdigit() else (1, part)) + return tuple(key) + + mlaunch = max(matches, key=_version_key) + getLogger().info("Resolved mlaunch: %s", mlaunch) + iOSHelper._mlaunch_path = mlaunch + return mlaunch + + # ── Device Detection ───────────────────────────────────────────── + + @staticmethod + def detect_connected_device(): + """Detect a connected physical iOS device and return its UDID. + + Checks IOS_DEVICE_UDID env var first, then auto-detects via devicectl. + Returns the UDID string, or None if no device is found. + """ + udid = os.environ.get('IOS_DEVICE_UDID', '').strip() + if udid: + getLogger().info("Using IOS_DEVICE_UDID from environment: %s", udid) + return udid + + getLogger().info("Auto-detecting connected iOS device...") + return iOSHelper._detect_via_devicectl_json() or iOSHelper._detect_via_devicectl_text() + + @staticmethod + def _detect_via_devicectl_json(): + """Try JSON-based device detection (Xcode 15+).""" + fd, json_tmp = tempfile.mkstemp(suffix='.json', prefix='devicectl_') + os.close(fd) + try: + result = subprocess.run( + ['xcrun', 'devicectl', 'list', 'devices', '--json-output', json_tmp], + capture_output=True, text=True, timeout=30 + ) + if result.returncode != 0: + return None + + with open(json_tmp, 'r') as f: + data = json.load(f) + + for device in data.get('result', {}).get('devices', []): + conn = device.get('connectionProperties', {}) + transport = conn.get('transportType', '') + # mlaunch uses the hardware UDID (e.g. 00008020-001965D83C43002E), + # NOT the CoreDevice identifier (a UUID like 5AE7F3E5-...). + # Fall back to the CoreDevice identifier if hw UDID is missing. + hw_udid = device.get('hardwareProperties', {}).get('udid', '') + device_udid = hw_udid or device.get('identifier', '') + if transport in ('wired', 'localNetwork', 'wifi') and device_udid: + name = device.get('deviceProperties', {}).get('name', 'unknown') + getLogger().info("Found device: %s (UDID: %s, transport: %s)", name, device_udid, transport) + return device_udid + return None + except (subprocess.TimeoutExpired, json.JSONDecodeError, KeyError, OSError): + return None + finally: + if os.path.exists(json_tmp): + os.remove(json_tmp) + + @staticmethod + def _detect_via_devicectl_text(): + """Fallback: parse device UDID from devicectl text output.""" + try: + result = subprocess.run( + ['xcrun', 'devicectl', 'list', 'devices'], + capture_output=True, text=True, timeout=30 + ) + if result.returncode != 0: + return None + + # Match CoreDevice UUID (8-4-4-4-12) or legacy Apple UDID formats + uuid_re = re.compile( + r'([0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}' + r'|[0-9A-Fa-f]{8}-[0-9A-Fa-f]{16}' + r'|[0-9A-Fa-f]{25,40})' + ) + for line in (result.stdout or '').splitlines(): + match = uuid_re.search(line) + if match: + getLogger().info("Detected device UDID: %s", match.group(1)) + return match.group(1) + return None + except Exception: + return None + + # ── Simulator UDID Resolution ──────────────────────────────────── + + @staticmethod + def _resolve_booted_simulator_udid(): + """Return the UDID of the simulator we should target. + + Resolution order: + 1. ``$HELIX_WORKITEM_ROOT/sim_udid.txt`` — written by setup_helix.py + after creating + booting our own per-workitem device. This pins + the UDID end-to-end so we never accidentally target a stale or + foreign device that happens to be booted on the machine. + 2. ``simctl list devices booted -j`` — fallback for local runs (no + Helix) where setup_helix.py was not invoked. + + mlaunch requires a real UDID (--device :v2:udid=); it does not + understand simctl's "booted" shortcut. + """ + # Prefer the pinned UDID written by setup_helix.py + workitem_root = os.environ.get('HELIX_WORKITEM_ROOT') + if workitem_root: + pinned = os.path.join(workitem_root, 'sim_udid.txt') + if os.path.isfile(pinned): + try: + with open(pinned, 'r', encoding='utf-8') as f: + udid = f.read().strip() + if udid: + getLogger().info("Using pinned simulator UDID from %s: %s", + pinned, udid) + return udid + except OSError as e: + getLogger().warning("Could not read %s: %s", pinned, e) + + try: + result = subprocess.run( + ['xcrun', 'simctl', 'list', 'devices', 'booted', '-j'], + capture_output=True, text=True, timeout=15 + ) + if result.returncode != 0: + return None + data = json.loads(result.stdout) + for runtime_devices in data.get('devices', {}).values(): + for dev in runtime_devices: + if dev.get('state', '').lower() == 'booted': + return dev['udid'] + return None + except Exception: + return None + + @staticmethod + def delete_simulator(udid): + """Shutdown and delete a simulator by UDID. Best-effort, never raises. + + Used by post.py to clean up the per-workitem simulator created in + setup_helix.py so devices don't accumulate across runs on the same + Helix machine. + """ + if not udid: + return + for cmd in (['xcrun', 'simctl', 'shutdown', udid], + ['xcrun', 'simctl', 'delete', udid]): + try: + subprocess.run(cmd, capture_output=True, text=True, timeout=60) + except Exception as e: + getLogger().warning("Cleanup '%s' failed: %s", ' '.join(cmd), e) + + # ── Unified Device Setup ───────────────────────────────────────── + + def setup_device(self, bundle_id, app_bundle_path, device_id='booted', is_physical=False): + """Prepare a simulator or physical device for testing. + + For simulators: ensures the device is booted and uninstalls any + existing app. For physical devices: stores the UDID for later use. + Does NOT install the app — call install_app() separately so + install timing can be captured independently. + """ + self.bundle_id = bundle_id + self.device_id = device_id + self.app_bundle_path = app_bundle_path + self.is_physical_device = is_physical + + if is_physical: + # Uninstall any stale app so first-deploy timing isn't affected + getLogger().info("Uninstalling any existing app (%s) from physical device: %s", bundle_id, device_id) + mlaunch = self._resolve_mlaunch() + self._run_quiet([mlaunch, '--uninstalldevbundleid', bundle_id, + '--devname', device_id]) + getLogger().info("Physical device setup complete: %s", device_id) + return + + # Simulator setup: boot if needed, uninstall stale app + if device_id != 'booted': + getLogger().info("Booting simulator: %s", device_id) + result = subprocess.run( + ['xcrun', 'simctl', 'boot', device_id], + capture_output=True, text=True + ) + if result.returncode != 0 and 'already booted' not in (result.stderr or '').lower(): + raise subprocess.CalledProcessError(result.returncode, result.args, result.stdout, result.stderr) + + # Resolve the actual UDID — mlaunch needs a real UDID, not "booted" + if self.device_id == 'booted': + resolved = self._resolve_booted_simulator_udid() + if not resolved: + raise RuntimeError( + "Could not resolve booted simulator UDID. " + "Ensure a simulator is booted (setup_helix.py should have done this).") + getLogger().info("Resolved booted simulator UDID: %s", resolved) + self.device_id = resolved + + self._run_quiet(['xcrun', 'simctl', 'uninstall', self.device_id, bundle_id]) + + # ── Device Code Signing ────────────────────────────────────────── + + def sign_app_for_device(self, app_bundle_path): + """Sign the .app bundle for physical device deployment. + + Mirrors the signing flow from maui_scenarios_ios.proj device startup: + 1. Copy embedded.mobileprovision into the .app bundle + 2. Run the Helix-provided 'sign' tool + + Both 'embedded.mobileprovision' and 'sign' are pre-installed on the + Mac.iPhone.17.Perf Helix machines. The build must use + EnableCodeSigning=false so MSBuild skips automatic signing. + + No-op for simulator builds or local runs where MSBuild handles signing + automatically (EnableCodeSigning is not disabled). + """ + if not self.is_physical_device: + return + + from performance.common import runninginlab + if not runninginlab(): + getLogger().info("Skipping post-build signing (local run — MSBuild handles signing)") + return + + import shutil + provision_src = 'embedded.mobileprovision' + provision_dst = os.path.join(app_bundle_path, 'embedded.mobileprovision') + + if not os.path.exists(provision_src): + getLogger().warning( + "embedded.mobileprovision not found in working directory. " + "Device signing may fail if the Helix machine doesn't have it.") + else: + shutil.copy2(provision_src, provision_dst) + getLogger().info("Copied provisioning profile into %s", app_bundle_path) + + app_name = os.path.basename(app_bundle_path) + app_dir = os.path.dirname(os.path.abspath(app_bundle_path)) + getLogger().info("Signing %s for device deployment", app_name) + + # Find the sign tool — it's pre-installed on Helix Mac machines + # but may not be on PATH in HelixWorkItem (vs XHarness) runners. + # Fast path: try direct resolution first, then fall back to running + # through a login shell (which is how Device Startup's XHarness + # CustomCommands find it — the login shell has the full PATH). + sign_cmd = shutil.which('sign') + if sign_cmd: + getLogger().info("Found sign tool on PATH: %s", sign_cmd) + RunCommand([sign_cmd, app_name], verbose=True).run(working_directory=app_dir) + return + + # Check known Helix machine locations + for candidate in ['/usr/local/bin/sign', + os.path.join(os.environ.get('HELIX_SCRIPT_ROOT', ''), 'sign')]: + if os.path.isfile(candidate) and os.access(candidate, os.X_OK): + getLogger().info("Found sign tool at known path: %s", candidate) + RunCommand([candidate, app_name], verbose=True).run(working_directory=app_dir) + return + + # Last resort: run through a login shell to get the full PATH. + # This mirrors how Device Startup's XHarness CustomCommands execute + # 'sign' — the XHarness runner's shell has the right PATH. + getLogger().info("sign tool not found on PATH or known locations; " + "trying login shell (bash -lc)") + shell_result = subprocess.run( + ['bash', '-lc', f'cd "{app_dir}" && sign "{app_name}"'], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + ) + if shell_result.stdout: + getLogger().info("sign output:\n%s", shell_result.stdout) + if shell_result.returncode != 0: + # The 'sign' tool only exists on Helix CI machines. For local + # device runs, MSBuild/Xcode already sign the app with the + # developer's identity during 'dotnet build', so re-signing is + # unnecessary. Warn instead of crashing so local measurement + # scripts (which set PERFLAB_INLAB=1 for reporting) can proceed. + getLogger().warning( + "'sign' tool not found — skipping re-signing. " + "App should already be signed by MSBuild for local device deployment. " + "(Tried: shutil.which('sign'), /usr/local/bin/sign, " + "HELIX_SCRIPT_ROOT/sign, bash -lc 'sign'. " + "Login shell exit code: %d)", + shell_result.returncode, + ) + return + getLogger().info("Signed %s via login shell successfully", app_name) + + # ── Unified Operations ─────────────────────────────────────────── + + def install_app(self, app_bundle_path): + """Install the app bundle and return wall-clock install time in ms. + + Per .NET iOS team guidance (Rolf), mlaunch is the canonical way to + deploy on both simulator and physical devices — it matches what the + IDEs do during F5, handles ad-hoc signing for personal-team device + deploys, and avoids `xcrun devicectl`'s strict CoreSign requirement + which fails for unsigned/ad-hoc Debug builds (MIInstallerErrorDomain + error 13 / "No code signature found"). + + Device: mlaunch --installdev --devname + Simulator: mlaunch --installsim --device :v2:udid= + """ + start = time.time() + mlaunch = self._resolve_mlaunch() + + if self.is_physical_device: + cmd = [mlaunch, '--installdev', app_bundle_path, + '--devname', self.device_id] + else: + cmd = [mlaunch, '--installsim', app_bundle_path, + '--device', f':v2:udid={self.device_id}'] + RunCommand(cmd, verbose=True).run() + + elapsed_ms = (time.time() - start) * 1000 + getLogger().info("Install completed in %.1f ms", elapsed_ms) + return elapsed_ms + + def measure_cold_startup(self, bundle_id): + """Measure app cold startup time in ms (int). + + - Simulator: ``xcrun simctl launch`` returns the launched PID + immediately. We measure wall-clock from invocation to PID + return (the same as what mlaunch internally reports), then + verify the process is still alive after a short stabilization + window so we don't report success for a crashed launch. + - Device: mlaunch --launchdev (returns immediately with PID) + + Note on the simulator path: an earlier version used + ``mlaunch --launchsim`` to better mirror the IDE F5 experience, + but on Apple Silicon Helix queues the simulator went from Booted + to Shutdown during/after that call (with no diagnostic output + from mlaunch). ``simctl launch`` is what mlaunch invokes + internally for the actual launch step, so the measurement is + equivalent for our purposes. + """ + + if self.is_physical_device: + return self._measure_device_startup_via_watchdog(bundle_id) + + # ── Simulator ──────────────────────────────────────────────── + # Sanity-check the simulator state before we start the timer so + # we fail fast with a clear error if the simulator has shut down + # (e.g. due to a previous workitem cleanup or system pressure). + self._assert_simulator_booted() + + # Verify the app is actually installed before timing the launch + # — otherwise an install-registration failure would be reported + # as a launch failure. + try: + container = subprocess.run( + ['xcrun', 'simctl', 'get_app_container', + self.device_id, bundle_id, 'app'], + capture_output=True, text=True, timeout=15, + ) + if container.returncode != 0: + raise RuntimeError( + f"App {bundle_id} is not installed in simulator " + f"{self.device_id} (simctl get_app_container exit " + f"{container.returncode}): {(container.stderr or '').strip()}" + ) + getLogger().info("App container: %s", (container.stdout or '').strip()) + except subprocess.TimeoutExpired: + raise RuntimeError( + f"simctl get_app_container timed out — simulator " + f"{self.device_id} not responding") + + # Terminate any running instance for a true cold start + self._run_quiet(['xcrun', 'simctl', 'terminate', self.device_id, bundle_id]) + time.sleep(0.5) + + # `simctl launch` returns immediately with the launched PID. + # `--terminate-running-process` makes the launch reliably cold + # even if termination above didn't take effect. + cmd = ['xcrun', 'simctl', 'launch', '--terminate-running-process', + self.device_id, bundle_id] + getLogger().info("$ %s", ' '.join(cmd)) + start = time.time() + result = subprocess.run(cmd, capture_output=True, text=True, timeout=180) + elapsed_ms = int((time.time() - start) * 1000) + + if result.returncode != 0: + # Dump diagnostics so we can tell whether the simulator died + # or the app failed to launch. + self._dump_simulator_diagnostics(bundle_id) + raise subprocess.CalledProcessError( + result.returncode, cmd, result.stdout, result.stderr) + + # `simctl launch` prints ": " on success. + pid = None + for token in (result.stdout or '').split(): + if token.isdigit(): + pid = int(token) + break + if pid is None: + getLogger().warning( + "Could not parse PID from simctl launch output: %r", + (result.stdout or '').strip()) + else: + getLogger().info("Launched %s with PID %d", bundle_id, pid) + # Stabilization check: the app should still be alive 2s + # later. If not, the launch was a crash, not a real start. + # + # iOS Simulator apps run as real macOS processes (sandboxed + # but in the host's process table — the PID returned by + # `simctl launch` IS the host PID), so we use host `ps -p` + # for the check. The simulator's userland does NOT include + # /bin/ps, so `simctl spawn ps` would fail with + # ENOENT regardless of whether the app is alive. + time.sleep(2.0) + check = subprocess.run( + ['ps', '-p', str(pid), '-o', 'pid='], + capture_output=True, text=True, timeout=15, + ) + if check.returncode != 0 or str(pid) not in (check.stdout or ''): + self._dump_simulator_diagnostics(bundle_id) + raise RuntimeError( + f"App {bundle_id} (PID {pid}) crashed within 2s of " + f"launch; host ps -p exit " + f"{check.returncode}, output: " + f"{(check.stdout or '').strip()!r}") + + getLogger().info("Cold startup: %d ms", elapsed_ms) + return elapsed_ms + + def _assert_simulator_booted(self): + """Raise RuntimeError if ``self.device_id`` is not in the Booted state.""" + try: + result = subprocess.run( + ['xcrun', 'simctl', 'list', 'devices', '-j'], + capture_output=True, text=True, timeout=15, + ) + except subprocess.TimeoutExpired: + raise RuntimeError("simctl list devices timed out") + try: + data = json.loads(result.stdout or '{}') + except Exception: + data = {} + for runtime_devices in (data.get('devices') or {}).values(): + for dev in runtime_devices: + if dev.get('udid') == self.device_id: + state = dev.get('state', '?') + if state != 'Booted': + raise RuntimeError( + f"Simulator {self.device_id} is in state " + f"{state!r}, expected 'Booted'. " + f"It must have been shut down between " + f"create_and_boot_simulator and the measurement.") + return + raise RuntimeError( + f"Simulator {self.device_id} not found in `simctl list devices`.") + + def _dump_simulator_diagnostics(self, bundle_id): + """Best-effort diagnostics dump for a failed simulator launch.""" + for cmd in ( + ['xcrun', 'simctl', 'list', 'devices', self.device_id], + ['xcrun', 'simctl', 'listapps', self.device_id], + # iOS sim has no /bin/ps; query the host process table for + # any descendants of the simulator hosting the bundle id. + ['ps', '-A', '-o', 'pid,ppid,command'], + ): + try: + r = subprocess.run(cmd, capture_output=True, text=True, timeout=15) + getLogger().error( + "$ %s\n--- exit %d ---\n%s\n--- stderr ---\n%s", + ' '.join(cmd), r.returncode, + (r.stdout or '')[-3000:], (r.stderr or '')[-1500:]) + except Exception as e: + getLogger().error("Diagnostic %s failed: %s", ' '.join(cmd), e) + + def _measure_device_startup_via_watchdog(self, bundle_id): + """Measure physical device cold startup using SpringBoard watchdog events. + + During every iOS app launch, SpringBoard emits four watchdog log events: + 1. "Now monitoring resource allowance of 20.00s" — OS starts loading the process + 2. "Stopped monitoring" — app reached main() + 3. "Now monitoring resource allowance of N.NNs" — OS waits for first frame + 4. "Stopped monitoring" — first frame drawn + + Time to Main = event2.timestamp - event1.timestamp + Time to First Draw = event4.timestamp - event3.timestamp + Total startup = Time to Main + Time to First Draw + + Requires sudo for `log collect --device`. + """ + # Give device a moment to settle before launch + time.sleep(1) + + # Record timestamp before launch for log collection window + start_ts = time.strftime('%Y-%m-%d %H:%M:%S%z') + + # Launch the app via mlaunch --launchdev (Rolf's guidance — same + # tool the IDEs use during F5; consistent with --installdev above). + # mlaunch --launchdev BLOCKS until the app exits (it tunnels stdout/ + # stderr from the device), so run it in a subprocess and terminate + # after we've collected enough log data. Killing mlaunch closes the + # tunnel but the app stays running on the device long enough for + # SpringBoard to emit the watchdog events we need. + mlaunch = self._resolve_mlaunch() + launch_cmd = [mlaunch, '--launchdev', self.app_bundle_path, + '--devname', self.device_id] + getLogger().info("$ %s", ' '.join(launch_cmd)) + launch_proc = subprocess.Popen( + launch_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + # Wait for the app to fully start before collecting logs. + time.sleep(5) + finally: + launch_proc.terminate() + try: + launch_proc.wait(timeout=5) + except subprocess.TimeoutExpired: + launch_proc.kill() + launch_proc.wait() + + # Collect device logs covering the launch window. + # Both rm -rf calls use sudo because `sudo log collect` writes the + # logarchive owned by root; an unprivileged rm cannot recurse into it + # and `log collect` refuses to overwrite an existing --output path + # (exit 74), so a stale archive from iteration N would block iteration + # N+1 from collecting any logs. + logarchive = os.path.join(tempfile.gettempdir(), 'ioshelper_startup.logarchive') + self._run_quiet(['sudo', 'rm', '-rf', logarchive]) + collect_cmd = ['sudo', 'log', 'collect', '--device', + '--start', start_ts, '--output', logarchive] + RunCommand(collect_cmd, verbose=True).run() + + # Parse SpringBoard watchdog events for this bundle ID. + # sudo because the logarchive is root-owned (see comment above). + show_cmd = ['sudo', 'log', 'show', + '--predicate', '(process == "SpringBoard") && (category == "Watchdog")', + '--info', '--style', 'ndjson', logarchive] + show = RunCommand(show_cmd, verbose=True) + show.run() + + events = [] + for line in show.stdout.splitlines(): + try: + data = json.loads(line) + msg = data.get('eventMessage', '') + if bundle_id not in msg: + continue + if 'Now monitoring resource allowance' in msg or 'Stopped monitoring' in msg: + events.append(data) + except (json.JSONDecodeError, KeyError): + continue + + if len(events) < 4: + getLogger().warning("Expected 4 watchdog events, got %d — falling back to wall-clock", len(events)) + # Couldn't parse watchdog events; return -1 to signal invalid measurement + return -1 + + # Sort by timestamp — log show output is not guaranteed chronological + events.sort(key=lambda evt: evt['timestamp']) + + # Validate expected sequence: monitor → stop → monitor → stop + expected = ['Now monitoring', 'Stopped monitoring', 'Now monitoring', 'Stopped monitoring'] + for i, keyword in enumerate(expected): + if keyword not in events[i].get('eventMessage', ''): + getLogger().warning("Unexpected watchdog event sequence at index %d: %s", i, events[i].get('eventMessage', '')) + return -1 + + # Parse timestamps: "2026-04-13 20:36:19.836430+0200" + def parse_ts(evt): + return datetime.strptime(evt['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z') + + t_main_start = parse_ts(events[0]) + t_main_end = parse_ts(events[1]) + t_draw_start = parse_ts(events[2]) + t_draw_end = parse_ts(events[3]) + + time_to_main_ms = int((t_main_end - t_main_start).total_seconds() * 1000) + time_to_draw_ms = int((t_draw_end - t_draw_start).total_seconds() * 1000) + total_ms = time_to_main_ms + time_to_draw_ms + + getLogger().info("Cold startup: %d ms (Time to Main: %d ms, Time to First Draw: %d ms)", + total_ms, time_to_main_ms, time_to_draw_ms) + + # Clean up logarchive (sudo: log collect ran as root, so the tree is root-owned) + self._run_quiet(['sudo', 'rm', '-rf', logarchive]) + + return total_ms + + def cleanup(self, skip_uninstall=False): + """Clean up the device session (simulator or physical). + + Device uses mlaunch --uninstalldevbundleid. Simulator keeps simctl + (mlaunch has no simulator terminate/uninstall commands). + """ + if skip_uninstall: + return + if self.is_physical_device: + mlaunch = self._resolve_mlaunch() + self._run_quiet([mlaunch, '--uninstalldevbundleid', self.bundle_id, + '--devname', self.device_id]) + else: + self._run_quiet(['xcrun', 'simctl', 'terminate', self.device_id, self.bundle_id]) + self._run_quiet(['xcrun', 'simctl', 'uninstall', self.device_id, self.bundle_id]) + + # ── App Bundle Discovery ───────────────────────────────────────── + + def find_app_bundle(self, build_output_dir, app_name, configuration='Debug', is_physical=False): + """Find the .app bundle in the build output directory. + + Searches for: bin//net*//.app + Returns the absolute path. Raises FileNotFoundError if not found. + """ + rid_patterns = ['ios-arm64'] if is_physical else ['iossimulator-*'] + for rid_pattern in rid_patterns: + pattern = os.path.join(build_output_dir, 'bin', configuration, 'net*', rid_pattern, f'{app_name}.app') + matches = glob.glob(pattern) + if matches: + if len(matches) > 1: + getLogger().warning("Multiple app bundles found: %s. Using first.", matches) + app_path = os.path.abspath(matches[0]) + getLogger().info("Found app bundle: %s", app_path) + return app_path + + raise FileNotFoundError( + f"No .app bundle in {build_output_dir}/bin/{configuration}/net*/{rid_patterns[0]}/{app_name}.app" + ) + + # ── Helpers ─────────────────────────────────────────────────────── + + @staticmethod + def _run_quiet(cmd): + """Run a command, suppressing CalledProcessError (best-effort).""" + try: + RunCommand(cmd, verbose=True).run() + except subprocess.CalledProcessError: + pass diff --git a/src/scenarios/shared/mauisharedpython.py b/src/scenarios/shared/mauisharedpython.py index 3b80dcb91cb..817b3a99a60 100644 --- a/src/scenarios/shared/mauisharedpython.py +++ b/src/scenarios/shared/mauisharedpython.py @@ -386,29 +386,49 @@ def __exit__(self, exc_type, exc_val, exc_tb): return False # Don't suppress exceptions +_ALL_MAUI_WORKLOADS = [ + "microsoft.net.sdk.android", + "microsoft.net.sdk.ios", + "microsoft.net.sdk.maccatalyst", + "microsoft.net.sdk.macos", + "microsoft.net.sdk.maui", + "microsoft.net.sdk.tvos", +] + def install_latest_maui( - precommands: PreCommands, - feed=extract_latest_dotnet_feed_from_nuget_config(path=os.path.join(get_repo_root_path(), "NuGet.config")) + precommands: PreCommands, + feed=None, + workloads=None, + workload_id=None, ): ''' - Install the latest maui workload using the provided feed. + Install the latest maui workload using the provided feed. This function will create a rollback file and install the maui workload using that file. + + Parameters: + precommands: PreCommands instance + feed: NuGet feed URL. If None, resolved from NuGet.config at call time. + workloads: List of manifest package IDs to resolve (e.g. ["microsoft.net.sdk.ios"]). + Defaults to all 6 MAUI workloads. + workload_id: CLI workload ID for 'dotnet workload install' (e.g. "maui-ios"). + Defaults to "maui". ''' + if feed is None: + feed = extract_latest_dotnet_feed_from_nuget_config( + path=os.path.join(get_repo_root_path(), "NuGet.config") + ) + if workloads is None: + workloads = _ALL_MAUI_WORKLOADS + if workload_id is None: + workload_id = 'maui' - getLogger().info("########## Installing latest MAUI workload ##########") + getLogger().info("########## Installing latest MAUI workload (%s) ##########", workload_id) if precommands.has_workload: getLogger().info("Skipping maui installation due to --has-workload=true") return - maui_rollback_dict: dict[str, str] = { - "microsoft.net.sdk.android" : "", - "microsoft.net.sdk.ios" : "", - "microsoft.net.sdk.maccatalyst" : "", - "microsoft.net.sdk.macos" : "", - "microsoft.net.sdk.maui" : "", - "microsoft.net.sdk.tvos" : "" - } + maui_rollback_dict: dict[str, str] = {w: "" for w in workloads} getLogger().info(f"Installing the latest maui workload from feed {feed}") @@ -514,6 +534,6 @@ def install_latest_maui( getLogger().info("Created rollback_maui.json file") # Install the workload using the rollback file - getLogger().info("Installing maui workload with rollback file") - precommands.install_workload('maui', ['--from-rollback-file', 'rollback_maui.json']) - getLogger().info("########## Finished installing latest MAUI workload ##########") + getLogger().info("Installing %s workload with rollback file", workload_id) + precommands.install_workload(workload_id, ['--from-rollback-file', 'rollback_maui.json']) + getLogger().info("########## Finished installing latest MAUI workload (%s) ##########", workload_id) diff --git a/src/scenarios/shared/precommands.py b/src/scenarios/shared/precommands.py index a7b6b7547a2..60491695f68 100644 --- a/src/scenarios/shared/precommands.py +++ b/src/scenarios/shared/precommands.py @@ -312,7 +312,14 @@ def get_packages_for_sdk_from_feed(self, sdk_name: str, feed: str): getLogger().debug(f"Raw packages response for {sdk_name}: {result.stdout}") try: - parsed_response = json.loads(result.stdout) + # Newer SDK versions (e.g., preview.4) may emit warnings or progress text + # before/after the JSON object in stdout. Extract only the JSON portion. + stdout_text = result.stdout + json_start = stdout_text.find('{') + json_end = stdout_text.rfind('}') + if json_start != -1 and json_end != -1 and json_end > json_start: + stdout_text = stdout_text[json_start:json_end + 1] + parsed_response = json.loads(stdout_text) getLogger().debug(f"Parsed JSON response for {sdk_name}: {parsed_response}") if not parsed_response.get("searchResult") or len(parsed_response["searchResult"]) == 0: diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index 165f0e882ba..0063be15fd8 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -6,6 +6,7 @@ import os import glob import re +import shlex import time import json @@ -18,6 +19,7 @@ from typing import Optional from shared.androidhelper import AndroidHelper from shared.androidinstrumentation import AndroidInstrumentationHelper +from shared.ioshelper import iOSHelper from shared.devicepowerconsumption import DevicePowerConsumptionHelper from shared.crossgen import CrossgenArguments from shared.startup import StartupWrapper @@ -31,6 +33,61 @@ from subprocess import CalledProcessError +# ── iOS Inner Loop report helpers ──────────────────────────────────── + +def _make_counter(name, metric, results, top=False): + """Create a performance counter dict for JSON reports.""" + return { + "name": name, + "topCounter": top, + "defaultCounter": False, + "higherIsBetter": False, + "metricName": metric, + "results": results, + } + + +def _merge_deploy_report(build_report_path, install_results, startup_results, output_path, app_size_bytes=None): + """Merge build metrics with install/startup timing into a single E2E report. + + If the build report doesn't exist (local runs without PERFLAB_INLAB=1), + creates a minimal report with only install and startup counters. + """ + if os.path.exists(build_report_path): + with open(build_report_path, 'r') as f: + report = json.load(f) + if not report.get("tests"): + report["tests"] = [{"counters": []}] + elif "counters" not in report["tests"][0]: + report["tests"][0]["counters"] = [] + else: + report = {"tests": [{"counters": []}]} + + report["tests"][0]["counters"].append(_make_counter("Install Time", "ms", install_results)) + report["tests"][0]["counters"].append(_make_counter("Cold Startup Time", "ms", startup_results, top=True)) + if app_size_bytes is not None: + report["tests"][0]["counters"].append(_make_counter("App Bundle Size", "bytes", [app_size_bytes], top=True)) + + with open(output_path, 'w') as f: + json.dump(report, f, indent=2) + + +def _measure_app_size(app_bundle_path): + """Return the total size of a .app bundle directory in bytes. + + Lightweight inline equivalent of the SizeOnDisk tool (SizeOnDisk.cs / sod.py) + which is designed for standalone size-focused scenarios with per-file breakdowns. + This helper just needs the total for a single counter in the E2E report. + """ + total = 0 + for dirpath, _dirnames, filenames in os.walk(app_bundle_path): + for f in filenames: + fp = os.path.join(dirpath, f) + if not os.path.islink(fp): + total += os.path.getsize(fp) + return total + + class Runner: ''' Wrapper for running all the things @@ -174,6 +231,20 @@ def parseargs(self): buildtimeparser.add_argument('--binlog-path', help='Location of binlog', dest='binlogpath') self.add_common_arguments(buildtimeparser) + iosinnerloopparser = subparsers.add_parser(const.IOSINNERLOOP, + description='measure first and incremental build+deploy time via binlogs (iOS)') + iosinnerloopparser.add_argument('--csproj-path', help='Path to .csproj file to build', dest='csprojpath') + iosinnerloopparser.add_argument('--edit-src', help='Modified source file paths, semicolon-separated', dest='editsrc') + iosinnerloopparser.add_argument('--edit-dest', help='Destination paths for modified files, semicolon-separated', dest='editdest') + iosinnerloopparser.add_argument('--framework', '-f', help='Target framework (e.g., net11.0-ios)', dest='framework') + iosinnerloopparser.add_argument('--configuration', '-c', help='Build configuration', dest='configuration', default='Debug') + iosinnerloopparser.add_argument('--msbuild-args', help='Additional MSBuild arguments', dest='msbuildargs', default='') + iosinnerloopparser.add_argument('--bundle-id', help='iOS bundle identifier', dest='bundleid') + iosinnerloopparser.add_argument('--device-id', help='iOS device ID (UDID for physical device, simulator ID or "booted" for simulator)', dest='deviceid', default='booted') + iosinnerloopparser.add_argument('--device-type', choices=['simulator', 'device'], help='Target device type: simulator (default) or physical device. Auto-detected from RuntimeIdentifier if not set.', dest='devicetype', default=None) + iosinnerloopparser.add_argument('--inner-loop-iterations', help='Number of incremental build+deploy+startup iterations (1+)', type=int, default=10, dest='innerloopiterations') + self.add_common_arguments(iosinnerloopparser) + args = parser.parse_args() if not args.testtype: @@ -196,6 +267,33 @@ def parseargs(self): if self.testtype == const.BUILDTIME: self.binlogpath = args.binlogpath + + if self.testtype == const.IOSINNERLOOP: + self.csprojpath = args.csprojpath + self.editsrcs = args.editsrc.split(';') if args.editsrc else [] + self.editdests = args.editdest.split(';') if args.editdest else [] + self.framework = args.framework + self.configuration = args.configuration + self.msbuildargs = args.msbuildargs or os.environ.get('PERFLAB_MSBUILD_ARGS', '') + # If IOS_RID is set (by .proj PreCommands or setup_helix.py arch + # detection), ensure RuntimeIdentifier in msbuildargs matches it. + ios_rid_env = os.environ.get('IOS_RID', '') + if ios_rid_env and 'RuntimeIdentifier=' in self.msbuildargs: + self.msbuildargs = re.sub( + r'RuntimeIdentifier=\S+', + f'RuntimeIdentifier={ios_rid_env}', + self.msbuildargs) + self.bundleid = args.bundleid + self.deviceid = args.deviceid + self.innerloopiterations = args.innerloopiterations + # Determine device type: explicit arg, or infer from RuntimeIdentifier + # in msbuildargs (ios-arm64 → device, iossimulator-* → simulator) + if args.devicetype: + self.devicetype = args.devicetype + elif 'RuntimeIdentifier=ios-arm64' in self.msbuildargs: + self.devicetype = 'device' + else: + self.devicetype = 'simulator' if self.testtype == const.DEVICESTARTUP: self.packagepath = args.packagepath @@ -974,4 +1072,270 @@ def run(self): if not (self.binlogpath and os.path.exists(os.path.join(const.TRACEDIR, self.binlogpath))): raise Exception("For build time measurements a valid binlog path must be provided.") self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.BUILDTIME, tracename=self.binlogpath, scenarioname=self.scenarioname) - startup.parsetraces(self.traits) \ No newline at end of file + startup.parsetraces(self.traits) + + elif self.testtype == const.IOSINNERLOOP: + from shutil import copy2, copytree + from performance.common import runninginlab + from performance.constants import UPLOAD_CONTAINER, UPLOAD_STORAGE_URI, UPLOAD_QUEUE + from shared.util import helixuploaddir + import upload + + # --- Validate inputs --- + if not self.csprojpath: + raise Exception("--csproj-path is required for iOS inner loop measurements.") + if not self.bundleid: + raise Exception("--bundle-id is required for iOS inner loop measurements.") + is_physical = (self.devicetype == 'device') + if is_physical and self.deviceid == 'booted': + detected = iOSHelper.detect_connected_device() + if not detected: + raise Exception("Physical device mode requires a device UDID. " + "Set --device-id or IOS_DEVICE_UDID, or connect a device.") + self.deviceid = detected + + # --- Cross-arch RID correction for simulator builds --- + # The .proj defaults RuntimeIdentifier to iossimulator-x64 because + # the original Helix queue (Mac.iPhone.17.Perf) was Intel x64. + # Other queues (e.g. Mac.iPhone.13.Perf) are Apple Silicon, where + # an x64 .app cannot install on an arm64 simulator (mlaunch fails + # with HE0046 "Failed to find matching arch"). Detect the host + # arch here and rewrite the RID in --msbuild-args so the build + # produces a binary matching the simulator we'll deploy to. + # Physical-device builds (ios-arm64) target the iPhone hardware, + # not the host, so we never override their RID. + if not is_physical and self.msbuildargs: + import platform as _platform + host_arch = _platform.machine() + if host_arch == 'arm64' and 'iossimulator-x64' in self.msbuildargs: + self.msbuildargs = self.msbuildargs.replace( + 'iossimulator-x64', 'iossimulator-arm64') + getLogger().info( + "Cross-arch RID correction: host=arm64, rewrote msbuild " + "args to use iossimulator-arm64 (was iossimulator-x64)") + elif host_arch == 'x86_64' and 'iossimulator-arm64' in self.msbuildargs: + self.msbuildargs = self.msbuildargs.replace( + 'iossimulator-arm64', 'iossimulator-x64') + getLogger().info( + "Cross-arch RID correction: host=x86_64, rewrote msbuild " + "args to use iossimulator-x64 (was iossimulator-arm64)") + # Keep IOS_RID env var aligned so versionmanager uses the same + # RID for the linked-DLL lookup below. + m = re.search(r'/p:RuntimeIdentifier=(\S+)', self.msbuildargs) + if m: + os.environ['IOS_RID'] = m.group(1) + + getLogger().info("iOS inner loop: device_type=%s, device_id=%s", self.devicetype, self.deviceid) + scenarioprefix = self.scenarioname or "MAUI iOS Build and Deploy" + + os.makedirs(const.TRACEDIR, exist_ok=True) + # Prefix binlog filenames with runtime flavor to avoid overwrites between runs + runtime_flavor = os.environ.get('RUNTIME_FLAVOR', '') + binlog_prefix = f'{runtime_flavor}-' if runtime_flavor else '' + first_binlog = os.path.join(const.TRACEDIR, f'{binlog_prefix}first-build-and-deploy.binlog') + + # Build base command (no -t:Install for iOS — plain dotnet build) + base_cmd = ['dotnet', 'build', self.csprojpath] + if self.configuration: + base_cmd.extend(['-c', self.configuration]) + if self.framework: + base_cmd.extend(['-f', self.framework]) + if self.msbuildargs: + base_cmd.extend([arg for arg in self.msbuildargs.split(';') if arg]) + + project_dir = os.path.dirname(os.path.abspath(self.csprojpath)) + exename = self.traits.exename + + # --- First build --- + try: + RunCommand(base_cmd + [f'-bl:{first_binlog}'], verbose=True).run() + except CalledProcessError: + getLogger().error("First build failed. Binlog: %s", first_binlog) + raise + + # --- Log SDK and workload versions --- + RunCommand(['dotnet', '--info'], verbose=True).run() + try: + from shared.versionmanager import versions_write_json, versions_write_env, get_sdk_versions + rid = 'ios-arm64' if is_physical else os.environ.get('IOS_RID', 'iossimulator-arm64') + linked_dir = os.path.join(project_dir, 'obj', self.configuration or 'Debug', + self.framework or 'net11.0-ios', rid, 'linked') + if os.path.isdir(linked_dir): + version_dict = get_sdk_versions(linked_dir, False) + versions_file = os.path.join(const.TRACEDIR, f'{binlog_prefix}versions.json') + versions_write_json(version_dict, versions_file) + versions_write_env(version_dict) + getLogger().info("SDK versions: %s", version_dict) + else: + getLogger().warning("Linked DLL dir not found at %s — skipping versions.json", linked_dir) + except Exception as e: + getLogger().warning("Could not extract SDK versions: %s", e) + + # --- Device setup + first deploy --- + iosHelper = iOSHelper() + edit_pairs = [] + try: + app_bundle = iosHelper.find_app_bundle(project_dir, exename, self.configuration, is_physical=is_physical) + first_app_size = _measure_app_size(app_bundle) + getLogger().info("App bundle size: %.2f MB (%d bytes)", first_app_size / 1048576, first_app_size) + iosHelper.setup_device(self.bundleid, app_bundle, self.deviceid, is_physical=is_physical) + iosHelper.sign_app_for_device(app_bundle) + first_install_ms = iosHelper.install_app(app_bundle) + first_startup_ms = iosHelper.measure_cold_startup(self.bundleid) + if first_startup_ms < 0: + raise RuntimeError("First deploy cold startup measurement failed (watchdog event parsing error)") + getLogger().info("First deploy: install=%.1f ms, startup=%d ms", first_install_ms, first_startup_ms) + + # Parse first build binlog + startup = StartupWrapper() + first_build_report = os.path.join(const.TRACEDIR, 'first-build-and-deploy-perf-lab-report.json') + startup.reportjson = first_build_report + saved_upload = self.traits.upload_to_perflab_container + self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.IOSINNERLOOP, + tracename=f'{binlog_prefix}first-build-and-deploy.binlog', + scenarioname=scenarioprefix + " - First Build and Deploy", + upload_to_perflab_container=False) + startup.parsetraces(self.traits) + + # Merge first build metrics + install/startup → first E2E report + first_e2e_report = os.path.join(const.TRACEDIR, 'first-debug-e2e-perf-lab-report.json') + _merge_deploy_report(first_build_report, [first_install_ms], [first_startup_ms], first_e2e_report, app_size_bytes=first_app_size) + + # --- Incremental loop --- + if not self.editsrcs or not self.editdests: + raise Exception("--edit-src and --edit-dest are required for incremental builds") + if len(self.editsrcs) != len(self.editdests): + raise Exception("--edit-src and --edit-dest must have the same number of semicolon-separated paths") + + for src, dest in zip(self.editsrcs, self.editdests): + with open(dest, 'r') as f: + original = f.read() + with open(src, 'r') as f: + modified = f.read() + edit_pairs.append((dest, original, modified)) + + num_iterations = self.innerloopiterations + getLogger().info("Starting %d incremental iterations", num_iterations) + + incremental_startup_results = [] + incremental_install_results = [] + incremental_app_size_results = [] + aggregated_counters = {} + report_template = None + + for iteration in range(1, num_iterations + 1): + getLogger().info("=== Incremental iteration %d/%d ===", iteration, num_iterations) + + # Toggle source files (odd → modified, even → original) + for dest, original, modified in edit_pairs: + content = modified if iteration % 2 == 1 else original + with open(dest, 'w') as f: + f.write(content) + + # Build + iter_binlog_name = '%sincremental-build-and-deploy-%d.binlog' % (binlog_prefix, iteration) + iter_binlog = os.path.join(const.TRACEDIR, iter_binlog_name) + try: + RunCommand(base_cmd + [f'-bl:{iter_binlog}'], verbose=True).run() + except CalledProcessError: + getLogger().error("Incremental build %d failed. Binlog: %s", iteration, iter_binlog) + raise + + # Sign (device only — no-op for simulator) + iosHelper.sign_app_for_device(app_bundle) + + # Measure app size after incremental build + iter_app_size = _measure_app_size(app_bundle) + getLogger().info("Iteration %d app bundle size: %.2f MB", iteration, iter_app_size / 1048576) + + # Install + startup + install_ms = iosHelper.install_app(app_bundle) + startup_ms = iosHelper.measure_cold_startup(self.bundleid) + if startup_ms < 0: + raise RuntimeError("Incremental deploy %d cold startup measurement failed (watchdog event parsing error)" % iteration) + getLogger().info("Iteration %d: install=%.1f ms, startup=%d ms", iteration, install_ms, startup_ms) + + incremental_install_results.append(install_ms) + incremental_startup_results.append(startup_ms) + incremental_app_size_results.append(iter_app_size) + + # Parse iteration binlog → temp report + iter_report = os.path.join(const.TRACEDIR, 'incremental-build-report-%d.json' % iteration) + startup.reportjson = iter_report + self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.IOSINNERLOOP, + tracename=iter_binlog_name, + scenarioname=scenarioprefix + " - Incremental Build and Deploy", + upload_to_perflab_container=False) + # Clear stale traces upload dir so copytree in parsetraces doesn't collide + helix_upload_dir = helixuploaddir() + if helix_upload_dir is not None: + traces_upload = os.path.join(helix_upload_dir, 'traces') + if os.path.exists(traces_upload): + rmtree(traces_upload) + + startup.parsetraces(self.traits) + + # Extract counters from temp report (may not exist on local runs) + if os.path.exists(iter_report): + with open(iter_report, 'r') as f: + iter_data = json.load(f) + test_obj = iter_data["tests"][0] + if report_template is None: + report_template = {k: v for k, v in test_obj.items() if k != "counters"} + for counter in test_obj["counters"]: + name = counter["name"] + if name not in aggregated_counters: + aggregated_counters[name] = { + "name": name, + "topCounter": counter.get("topCounter", False), + "defaultCounter": counter.get("defaultCounter", False), + "higherIsBetter": counter.get("higherIsBetter", False), + "metricName": counter.get("metricName", "ms"), + "results": [] + } + aggregated_counters[name]["results"].extend(counter.get("results", [])) + os.remove(iter_report) + + # --- Build final incremental report --- + incremental_e2e_report = os.path.join(const.TRACEDIR, 'incremental-debug-e2e-perf-lab-report.json') + final_counters = list(aggregated_counters.values()) + final_counters.append(_make_counter("Install Time", "ms", incremental_install_results)) + final_counters.append(_make_counter("Cold Startup Time", "ms", incremental_startup_results, top=True)) + final_counters.append(_make_counter("App Bundle Size", "bytes", incremental_app_size_results, top=True)) + + final_test = dict(report_template or {}) + final_test["counters"] = final_counters + with open(incremental_e2e_report, 'w') as f: + json.dump({"tests": [final_test]}, f, indent=2) + + # --- Persist reports for local runs --- + results_dir = os.path.join(os.getcwd(), 'results', os.environ.get('RUNTIME_FLAVOR', 'unknown')) + try: + os.makedirs(results_dir, exist_ok=True) + for report in [first_e2e_report, incremental_e2e_report]: + copy2(report, os.path.join(results_dir, os.path.basename(report))) + except Exception as e: + getLogger().warning("Failed to persist reports: %s", e) + + # --- Upload to Helix --- + self.traits.add_traits(overwrite=True, upload_to_perflab_container=saved_upload) + helix_upload_dir = helixuploaddir() + if runninginlab() and helix_upload_dir is not None: + traces_upload = os.path.join(helix_upload_dir, 'traces') + if os.path.exists(traces_upload): + rmtree(traces_upload) + copytree(const.TRACEDIR, traces_upload, dirs_exist_ok=True) + if self.traits.upload_to_perflab_container: + for report_path in [first_e2e_report, incremental_e2e_report]: + upload_code = upload.upload(report_path, UPLOAD_CONTAINER, UPLOAD_QUEUE, UPLOAD_STORAGE_URI) + if upload_code != 0: + sys.exit(upload_code) + + finally: + for dest, original, _modified in edit_pairs: + try: + with open(dest, 'w') as f: + f.write(original) + except Exception as e: + getLogger().warning("Failed to restore %s: %s", dest, e) + iosHelper.cleanup(skip_uninstall=True) diff --git a/src/tools/ScenarioMeasurement/Startup/Startup.cs b/src/tools/ScenarioMeasurement/Startup/Startup.cs index 3c8a180561b..536339f04d6 100644 --- a/src/tools/ScenarioMeasurement/Startup/Startup.cs +++ b/src/tools/ScenarioMeasurement/Startup/Startup.cs @@ -27,6 +27,7 @@ enum MetricType WinUIBlazor, TimeToMain2, BuildTime, + IOSInnerLoop, } public class InnerLoopMarkerEventSource : EventSource @@ -291,6 +292,7 @@ static void checkArg(string arg, string name) MetricType.WinUIBlazor => new WinUIBlazorParser(), MetricType.TimeToMain2 => new TimeToMain2Parser(AddTestProcessEnvironmentVariable), MetricType.BuildTime => new BuildTimeParser(), + MetricType.IOSInnerLoop => new IOSInnerLoopParser(), _ => throw new ArgumentOutOfRangeException(), }; diff --git a/src/tools/ScenarioMeasurement/Util/Parsers/IOSInnerLoopParser.cs b/src/tools/ScenarioMeasurement/Util/Parsers/IOSInnerLoopParser.cs new file mode 100644 index 00000000000..f78f765255c --- /dev/null +++ b/src/tools/ScenarioMeasurement/Util/Parsers/IOSInnerLoopParser.cs @@ -0,0 +1,115 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Build.Logging.StructuredLogger; +using StructuredLogViewer; +using Microsoft.Diagnostics.Tracing; +using Reporting; + +namespace ScenarioMeasurement; + +/// +/// Parses iOS inner loop (build+deploy) target and task durations from a binary log file. +/// +public class IOSInnerLoopParser : IParser +{ + public void EnableKernelProvider(ITraceSession kernel) { throw new NotImplementedException(); } + public void EnableUserProviders(ITraceSession user) { throw new NotImplementedException(); } + + // Task name (case-insensitive) → counter display name + private static readonly Dictionary TrackedTasks = new(StringComparer.OrdinalIgnoreCase) + { + // Shared build tasks + ["Csc"] = "Csc Task Time", + ["XamlCTask"] = "XamlC Task Time", + ["LinkAssembliesNoShrink"] = "LinkAssembliesNoShrink Task Time", + ["FilterAssemblies"] = "FilterAssemblies Task Time", + ["ResolveSdks"] = "ResolveSdks Task Time", + ["ProcessAssemblies"] = "ProcessAssemblies Task Time", + ["GenerateNativeApplicationConfigSources"] = "GenerateNativeApplicationConfigSources Task Time", + // iOS-specific build tasks + ["AOTCompile"] = "AOTCompile Task Time", + ["MonoAOTCompiler"] = "MonoAOTCompiler Task Time", + ["ILLink"] = "ILLink Task Time", + ["CompileNativeCode"] = "CompileNativeCode Task Time", + ["GetFileHash"] = "GetFileHash Task Time", + ["Codesign"] = "Codesign Task Time", + ["CompileNativeFiles"] = "CompileNativeFiles Task Time", + ["LinkNativeCode"] = "LinkNativeCode Task Time", + ["GenerateBundleName"] = "GenerateBundleName Task Time", + ["CreateAssetPack"] = "CreateAssetPack Task Time", + ["ComputeCodesignInputs"] = "ComputeCodesignInputs Task Time", + ["DetectSigningIdentity"] = "DetectSigningIdentity Task Time", + ["CompileAppManifest"] = "CompileAppManifest Task Time", + ["CompileEntitlements"] = "CompileEntitlements Task Time", + ["CreateBindingResourcePackage"] = "CreateBindingResourcePackage Task Time", + ["MTouch"] = "MTouch Task Time", + ["ACTool"] = "ACTool Task Time", + ["IBTool"] = "IBTool Task Time", + ["DSymUtil"] = "DSymUtil Task Time", + }; + + // Target name (case-sensitive, matches MSBuild conventions) → counter display name + private static readonly Dictionary TrackedTargets = new(StringComparer.Ordinal) + { + ["CoreCompile"] = "CoreCompile Target Time", + ["XamlC"] = "XamlC Target Time", + ["_AOTCompile"] = "_AOTCompile Target Time", + ["_CodesignAppBundle"] = "_CodesignAppBundle Target Time", + ["_CompileToNative"] = "_CompileToNative Target Time", + ["_RunILLink"] = "_RunILLink Target Time", + ["_SelectR2RAssemblies"] = "_SelectR2RAssemblies Target Time", + ["_LinkR2RFramework"] = "_LinkR2RFramework Target Time", + ["_CompileNativeExecutable"] = "_CompileNativeExecutable Target Time", + ["_GenerateDSym"] = "_GenerateDSym Target Time", + ["_CreateAppBundle"] = "_CreateAppBundle Target Time", + ["_CopyResourcesToBundle"] = "_CopyResourcesToBundle Target Time", + ["_GenerateBundleName"] = "_GenerateBundleName Target Time", + }; + + public IEnumerable Parse(string binlogFile, string processName, IList pids, string commandLine) + { + if (!File.Exists(binlogFile)) + yield break; + + var build = BinaryLog.ReadBuild(binlogFile); + BuildAnalyzer.AnalyzeBuild(build); + + // Collect task durations + var taskResults = TrackedTasks.ToDictionary(kv => kv.Key, _ => new List(), StringComparer.OrdinalIgnoreCase); + foreach (var task in build.FindChildrenRecursive()) + { + if (taskResults.TryGetValue(task.Name, out var list)) + list.Add(task.Duration.TotalMilliseconds / 1000.0); + } + + // Collect target durations + var targetResults = TrackedTargets.ToDictionary(kv => kv.Key, _ => new List(), StringComparer.Ordinal); + foreach (var target in build.FindChildrenRecursive()) + { + if (targetResults.TryGetValue(target.Name, out var list)) + list.Add(target.Duration.TotalMilliseconds / 1000.0); + } + + // Overall build duration + yield return new Counter { Name = "Build Time", MetricName = "s", DefaultCounter = true, TopCounter = true, + Results = new[] { build.Duration.TotalMilliseconds / 1000.0 } }; + + // Emit task counters + foreach (var (taskName, counterName) in TrackedTasks) + { + var results = taskResults[taskName]; + if (results.Count > 0) + yield return new Counter { Name = counterName, MetricName = "s", DefaultCounter = false, TopCounter = true, Results = results.ToArray() }; + } + + // Emit target counters + foreach (var (targetName, counterName) in TrackedTargets) + { + var results = targetResults[targetName]; + if (results.Count > 0) + yield return new Counter { Name = counterName, MetricName = "s", DefaultCounter = false, TopCounter = true, Results = results.ToArray() }; + } + } +}