Add MAUI Android inner loop deploy measurement scenario

eng/performance/maui_scenarios_android.proj

@@ -4,9 +4,10 @@
 
   <PropertyGroup>
     <IncludeXHarnessCli>true</IncludeXHarnessCli>
+    <_IsInnerLoop Condition="'$(RunKind)' == 'maui_scenarios_android_innerloop'">true</_IsInnerLoop>
   </PropertyGroup>
 
-  <PropertyGroup>
+  <PropertyGroup Condition="'$(_IsInnerLoop)' != 'true'">
     <AfterPreparePayloadWorkItemCommand>$(Python) post.py</AfterPreparePayloadWorkItemCommand>
     <PreparePayloadOutDirectoryName>scenarios_out</PreparePayloadOutDirectoryName>
     <PreparePayloadWorkItemBaseDirectory Condition="'$(TargetsWindows)' == 'true'">$(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\</PreparePayloadWorkItemBaseDirectory>
@@ -34,16 +35,17 @@
 
   <Target Name="RemoveDotnetFromCorrelationStaging" BeforeTargets="BeforeTest">
     <Message Text="Removing Dotnet Packs from Correlation Staging" Importance="high" />
-    <RemoveDir Directories="$(CorrelationPayloadDirectory)dotnet\packs" />
+    <RemoveDir Directories="$(CorrelationPayloadDirectory)dotnet\packs" Condition="'$(TargetsWindows)' == 'true'" />
+    <RemoveDir Directories="$(CorrelationPayloadDirectory)dotnet/packs" Condition="'$(TargetsWindows)' != 'true'" />
   </Target>
 
-  <ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(_IsInnerLoop)' != 'true'">
     <HelixWorkItem>
       <Timeout>00:30</Timeout>
     </HelixWorkItem>
   </ItemDefinitionGroup>
 
-  <ItemGroup>
+  <ItemGroup Condition="'$(_IsInnerLoop)' != 'true'">
     <MAUIAndroidScenario Include=".NET Android Default Template">
       <ScenarioDirectoryName>netandroid</ScenarioDirectoryName>
       <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
@@ -72,7 +74,7 @@
   </ItemGroup>
 
 
-  <ItemGroup>
+  <ItemGroup Condition="'$(_IsInnerLoop)' != 'true'">
     <PreparePayloadWorkItem Include="@(MAUIAndroidScenario)">
       <Command>$(Python) pre.py publish -f $(PERFLAB_Framework)-android -r android-arm64 --self-contained -c $(BuildConfig) --msbuild=&quot;$(_MSBuildArgs)&quot; --binlog $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)\%(PreparePayloadWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)</Command>
       <WorkingDirectory>%(PreparePayloadWorkItem.PayloadDirectory)</WorkingDirectory>
@@ -81,7 +83,7 @@
 
 
 <!-- We only run the android tests from Windows machines (at least for now) -->
-  <ItemGroup>
+  <ItemGroup Condition="'$(_IsInnerLoop)' != 'true'">
     <HelixWorkItem Include="@(MAUIAndroidScenario -> 'SOD - %(Identity) APK Size')" Condition="!$(HelixTargetQueue.ToLowerInvariant().Contains('pixel'))">
       <PreCommands>echo on; xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName) %HELIX_WORKITEM_ROOT%\pub\ /E /I /Y</PreCommands>
       <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot; $(ScenarioArgs)</Command>
@@ -116,6 +118,73 @@
     </HelixWorkItem>
   </ItemGroup>
 
+  <PropertyGroup Condition="'$(_IsInnerLoop)' == 'true'">
+    <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'mono'">/p:UseMonoRuntime=true</_MSBuildArgs>
+    <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr'">/p:UseMonoRuntime=false</_MSBuildArgs>
+    <!-- Disable ReadyToRun so inner loop measures JIT behavior. -->
+    <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr'">$(_MSBuildArgs);/p:PublishReadyToRun=false;/p:PublishReadyToRunComposite=false</_MSBuildArgs>
+
+    <!-- Windows targets physical devices; Linux targets emulators. -->
+    <AndroidRid Condition="'$(AndroidRid)' == '' and '$(TargetsWindows)' == 'true'">android-arm64</AndroidRid>
+    <AndroidRid Condition="'$(AndroidRid)' == '' and '$(TargetsWindows)' != 'true'">android-x64</AndroidRid>
+    <_MSBuildArgs>$(_MSBuildArgs) /p:RuntimeIdentifier=$(AndroidRid)</_MSBuildArgs>
+
+    <!-- TODO: https://github.com/dotnet/maui/issues/34706 -->
+    <_MSBuildArgs>$(_MSBuildArgs) /p:SupportedOSPlatformVersion=23</_MSBuildArgs>
+    <!-- Force a single android TFM so dotnet build's implicit restore skips multi-platform TFMs. -->
+    <_MSBuildArgs Condition="!$(_MSBuildArgs.Contains('/p:TargetFrameworks='))">$(_MSBuildArgs) /p:TargetFrameworks=$(PERFLAB_Framework)-android</_MSBuildArgs>
+
+    <RunConfigsString>$(RuntimeFlavor)_$(CodegenType)</RunConfigsString>
+    <InnerLoopIterations Condition="'$(InnerLoopIterations)' == ''">10</InnerLoopIterations>
+    <!-- Must outlast the full scenario; otherwise logcat never emits 'Displayed'. -->
+    <ScreenTimeoutMs Condition="'$(ScreenTimeoutMs)' == ''">1800000</ScreenTimeoutMs>
+  </PropertyGroup>
+
+  <ItemDefinitionGroup Condition="'$(_IsInnerLoop)' == 'true'">
+    <HelixWorkItem>
+      <Timeout>01:00</Timeout>
+    </HelixWorkItem>
+  </ItemDefinitionGroup>
+
+  <ItemGroup Condition="'$(_IsInnerLoop)' == 'true'">
+    <MAUIAndroidInnerLoopScenario Include="MAUI Android Inner Loop">
+      <ScenarioDirectoryName>mauiandroidinnerloop</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+    </MAUIAndroidInnerLoopScenario>
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(_IsInnerLoop)' == 'true'">
+    <PreparePayloadWorkItem Include="@(MAUIAndroidInnerLoopScenario)">
+      <Command>$(Python) pre.py default -f $(PERFLAB_Framework)</Command>
+      <WorkingDirectory>%(PreparePayloadWorkItem.PayloadDirectory)</WorkingDirectory>
+    </PreparePayloadWorkItem>
+  </ItemGroup>
+
+  <!-- Helix env vars set before setup_helix.py so PATH includes adb. -->
+  <PropertyGroup Condition="'$(_IsInnerLoop)' == 'true'">
+    <_WindowsEnvVars>set DOTNET_ROOT=%HELIX_CORRELATION_PAYLOAD%\dotnet;set DOTNET_CLI_TELEMETRY_OPTOUT=1;set DOTNET_MULTILEVEL_LOOKUP=0;set NUGET_PACKAGES=%HELIX_WORKITEM_ROOT%\.packages;set ANDROID_HOME=%HELIX_WORKITEM_ROOT%\android-sdk;set ANDROID_SDK_ROOT=%HELIX_WORKITEM_ROOT%\android-sdk;set JAVA_HOME=%HELIX_WORKITEM_ROOT%\jdk;set PATH=%HELIX_CORRELATION_PAYLOAD%\dotnet;;%HELIX_WORKITEM_ROOT%\android-sdk\platform-tools;;%HELIX_WORKITEM_ROOT%\jdk\bin;;%PATH%</_WindowsEnvVars>
+    <_LinuxEnvVars>export DOTNET_ROOT=$HELIX_CORRELATION_PAYLOAD/dotnet;export DOTNET_CLI_TELEMETRY_OPTOUT=1;export DOTNET_MULTILEVEL_LOOKUP=0;export NUGET_PACKAGES=$HELIX_WORKITEM_ROOT/.packages;export ANDROID_HOME=$HELIX_WORKITEM_ROOT/android-sdk;export ANDROID_SDK_ROOT=$HELIX_WORKITEM_ROOT/android-sdk;export JAVA_HOME=$HELIX_WORKITEM_ROOT/jdk;export ANDROID_SERIAL=emulator-5554;export PATH=$HELIX_CORRELATION_PAYLOAD/dotnet:$HELIX_WORKITEM_ROOT/android-sdk/platform-tools:$HELIX_WORKITEM_ROOT/jdk/bin:$PATH</_LinuxEnvVars>
+  </PropertyGroup>
+
+  <!-- Inner loop work items hardcode -c Debug; Release is not a supported configuration here. -->
+  <ItemGroup Condition="'$(_IsInnerLoop)' == 'true' and '$(TargetsWindows)' == 'true'">
+    <HelixWorkItem Include="@(MAUIAndroidInnerLoopScenario -> 'Inner Loop Device - %(Identity)')">
+      <PreCommands>$(_WindowsEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-android &quot;$(_MSBuildArgs)&quot;</PreCommands>
+      <Command>$(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src &quot;src/MainPage.xaml.cs;src/MainPage.xaml&quot; --edit-dest &quot;app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml&quot; --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c Debug --msbuild-args &quot;$(_MSBuildArgs)&quot; --scenario-name &quot;%(Identity)&quot; --inner-loop-iterations $(InnerLoopIterations) --screen-timeout-ms $(ScreenTimeoutMs) $(ScenarioArgs)</Command>
+      <PostCommands>$(Python) post.py</PostCommands>
+      <DownloadFilesFromResults>output.log</DownloadFilesFromResults>
+    </HelixWorkItem>
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(_IsInnerLoop)' == 'true' and '$(TargetsWindows)' != 'true'">
+    <HelixWorkItem Include="@(MAUIAndroidInnerLoopScenario -> 'Inner Loop Emulator - %(Identity)')">
+      <PreCommands>$(_LinuxEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-android &quot;$(_MSBuildArgs)&quot;</PreCommands>
+      <Command>$(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src &quot;src/MainPage.xaml.cs;src/MainPage.xaml&quot; --edit-dest &quot;app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml&quot; --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c Debug --msbuild-args &quot;$(_MSBuildArgs)&quot; --scenario-name &quot;%(Identity)&quot; --inner-loop-iterations $(InnerLoopIterations) --screen-timeout-ms $(ScreenTimeoutMs) $(ScenarioArgs)</Command>
+      <PostCommands>$(Python) post.py</PostCommands>
+      <DownloadFilesFromResults>output.log</DownloadFilesFromResults>
+    </HelixWorkItem>
+  </ItemGroup>
+
   <Import Project="PreparePayloadWorkItems.targets" />
 
 </Project>

eng/pipelines/sdk-perf-jobs.yml

@@ -527,6 +527,84 @@ jobs:
         ${{ each parameter in parameters.jobParameters }}:
           ${{ parameter.key }}: ${{ parameter.value }}
 
+  # Maui Android inner loop benchmarks on devices (Mono Default) - Debug
+  - template: /eng/pipelines/templates/build-machine-matrix.yml
+    parameters:
+      jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml
+      buildMachines:
+        - win-x64-android-arm64-pixel
+        - win-x64-android-arm64-galaxy
+      isPublic: false
+      jobParameters:
+        runKind: maui_scenarios_android_innerloop
+        projectFileName: maui_scenarios_android.proj
+        channels:
+          - main
+        runtimeFlavor: mono
+        codeGenType: Default
+        buildConfig: Debug
+        additionalJobIdentifier: Mono_Debug_InnerLoop
+        ${{ each parameter in parameters.jobParameters }}:
+          ${{ parameter.key }}: ${{ parameter.value }}
+
+  # Maui Android inner loop benchmarks on devices (CoreCLR Default) - Debug
+  - template: /eng/pipelines/templates/build-machine-matrix.yml
+    parameters:
+      jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml
+      buildMachines:
+        - win-x64-android-arm64-pixel
+        - win-x64-android-arm64-galaxy
+      isPublic: false
+      jobParameters:
+        runKind: maui_scenarios_android_innerloop
+        projectFileName: maui_scenarios_android.proj
+        channels:
+          - main
+        runtimeFlavor: coreclr
+        codeGenType: Default
+        buildConfig: Debug
+        additionalJobIdentifier: CoreCLR_Debug_InnerLoop
+        ${{ each parameter in parameters.jobParameters }}:
+          ${{ parameter.key }}: ${{ parameter.value }}
+
+  # Maui Android inner loop benchmarks on emulator (Mono Default) - Debug
+  - template: /eng/pipelines/templates/build-machine-matrix.yml
+    parameters:
+      jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml
+      buildMachines:
+        - ubuntu-x64-android-emulator
+      isPublic: false
+      jobParameters:
+        runKind: maui_scenarios_android_innerloop
+        projectFileName: maui_scenarios_android.proj
+        channels:
+          - main
+        runtimeFlavor: mono
+        codeGenType: Default
+        buildConfig: Debug
+        additionalJobIdentifier: Mono_Debug_InnerLoop_Emulator
+        ${{ each parameter in parameters.jobParameters }}:
+          ${{ parameter.key }}: ${{ parameter.value }}
+
+  # Maui Android inner loop benchmarks on emulator (CoreCLR Default) - Debug
+  - template: /eng/pipelines/templates/build-machine-matrix.yml
+    parameters:
+      jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml
+      buildMachines:
+        - ubuntu-x64-android-emulator
+      isPublic: false
+      jobParameters:
+        runKind: maui_scenarios_android_innerloop
+        projectFileName: maui_scenarios_android.proj
+        channels:
+          - main
+        runtimeFlavor: coreclr
+        codeGenType: Default
+        buildConfig: Debug
+        additionalJobIdentifier: CoreCLR_Debug_InnerLoop_Emulator
+        ${{ each parameter in parameters.jobParameters }}:
+          ${{ parameter.key }}: ${{ parameter.value }}
+
   # Maui iOS scenario benchmarks (Mono - Default) - Debug
   - template: /eng/pipelines/templates/build-machine-matrix.yml
     parameters:

eng/pipelines/templates/build-machine-matrix.yml

@@ -146,6 +146,19 @@ jobs:
       machinePool: GalaxyA16
       ${{ insert }}: ${{ parameters.jobParameters }}
 
+- ${{ if and(containsValue(parameters.buildMachines, 'ubuntu-x64-android-emulator'), not(eq(parameters.isPublic, true))) }}: # Ubuntu x64 Android emulator only used in private builds currently
+  - template: ${{ parameters.jobTemplate }}
+    parameters:
+      osGroup: ubuntu
+      archType: x64
+      osVersion: 2204
+      pool:
+        vmImage: ubuntu-latest
+      container: ubuntu_x64_build_container
+      queue: Ubuntu.2204.Amd64.Android.36
+      machinePool: AndroidEmulator
+      ${{ insert }}: ${{ parameters.jobParameters }}
+
 - ${{ if and(containsValue(parameters.buildMachines, 'osx-x64-ios-arm64'), not(eq(parameters.isPublic, true))) }}: # iPhone ARM64 17 only used in private builds currently
   - template: ${{ parameters.jobTemplate }}
     parameters:

eng/pipelines/templates/run-performance-job.yml

@@ -190,7 +190,7 @@ jobs:
               - '--is-scenario'
             - ${{ if ne(length(parameters.runEnvVars), 0) }}:
               - "--run-env-vars ${{ join(' ', parameters.runEnvVars)}}"
-            - ${{ if and(in(parameters.runKind, 'maui_scenarios_ios', 'maui_scenarios_android'), ne(parameters.runtimeFlavor, '')) }}:
+            - ${{ if and(in(parameters.runKind, 'maui_scenarios_ios', 'maui_scenarios_android', 'maui_scenarios_android_innerloop'), ne(parameters.runtimeFlavor, '')) }}:
               - '--runtime-flavor ${{ parameters.runtimeFlavor }}'
             - ${{ if ne(parameters.osVersion, '') }}:
               - '--os-version ${{ parameters.osVersion }}'
@@ -245,6 +245,8 @@ jobs:
         parameters:
           osGroup: ${{ parameters.osGroup }}
           projectFile: $(_projectFile)
+          environment:
+            RunKind: ${{ parameters.runKind }}
       - ${{ if eq(parameters.osGroup, 'windows') }}:
         - task: PowerShell@2
           displayName: Redact Logs

scripts/run_performance_job.py

@@ -570,9 +570,9 @@ def get_run_configurations(
             configurations["iOSLlvmBuild"] = str(ios_llvm_build)
 
     # .NET Android and .NET MAUI Android sample app scenarios
-    if run_kind == "maui_scenarios_android":
+    if run_kind in ["maui_scenarios_android", "maui_scenarios_android_innerloop"]:
         if not runtime_flavor in ("mono", "coreclr"):
-            raise Exception("Runtime flavor must be specified for maui_scenarios_android")
+            raise Exception(f"Runtime flavor must be specified for {run_kind}")
         configurations["CodegenType"] = str(codegen_type)
         configurations["RuntimeType"] = str(runtime_flavor)
         if build_config is not None and build_config != DEFAULT_BUILD_CONFIG:
@@ -1160,6 +1160,7 @@ def publish_dotnet_app_to_payload(payload_dir_name: str, csproj_path: str, self_
             os.environ["RuntimeFlavor"] = args.runtime_flavor or ''
             os.environ["CodegenType"] = args.codegen_type or ''
             os.environ["BuildConfig"] = args.build_config or DEFAULT_BUILD_CONFIG
+            os.environ["RunKind"] = args.run_kind or ''
 
             # TODO: See if these commands are needed for linux as they were being called before but were failing.
             if args.os_group == "windows" or args.os_group == "osx":
@@ -1190,7 +1191,7 @@ def publish_dotnet_app_to_payload(payload_dir_name: str, csproj_path: str, self_
                 verbose=True).run()
 
             # Search for additional binlogs generated by the maui scenarios prepare payload work items to copy to the artifacts log dir
-            if args.run_kind in ["maui_scenarios_android", "maui_scenarios_ios"]:
+            if args.run_kind in ["maui_scenarios_android", "maui_scenarios_ios", "maui_scenarios_android_innerloop"]:
                 for binlog_path in glob(os.path.join(payload_dir, "scenarios_out", "**", "*.binlog"), recursive=True):
                     shutil.copy(binlog_path, ci_artifacts_log_dir)
 

src/scenarios/mauiandroidinnerloop/post.py

@@ -0,0 +1,28 @@
+'''
+post cleanup script
+'''
+
+import subprocess
+import sys
+import traceback
+from performance.logger import setup_loggers, getLogger
+from shared.postcommands import clean_directories
+from shared.util import xharness_adb
+from test import EXENAME
+
+setup_loggers(True)
+logger = getLogger(__name__)
+
+try:
+    # Uninstall the app from the connected device so re-runs start from a clean state
+    package_name = f'com.companyname.{EXENAME.lower()}'
+    logger.info(f"Uninstalling {package_name} from device")
+    subprocess.run(xharness_adb() + ['uninstall', package_name], check=False)
+
+    logger.info("Shutting down dotnet build servers")
+    subprocess.run(['dotnet', 'build-server', 'shutdown'], check=False)
+
+    clean_directories()
+except Exception as e:
+    logger.error(f"Post cleanup failed: {e}\n{traceback.format_exc()}")
+    sys.exit(1)