ci(benchmark): benchmark generation worflow (WPB-24729)

.github/workflows/generate-baseline-profile.yml

@@ -0,0 +1,127 @@
+name: "Generate Baseline Profile"
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 2 * * 0"
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: generate-baseline-profile
+  cancel-in-progress: false
+
+env:
+  GITHUB_TOKEN: ${{ secrets.ANDROID_BOB_GH_TOKEN }}
+  BASE_BRANCH: develop
+  REFRESH_BRANCH: chore/baseline-profile-refresh
+  OP_VAULT: "Test Automation"
+  TARGET_PACKAGE: com.wire
+
+jobs:
+  generate:
+    name: Generate baseline profile (prod compatrelease)
+    runs-on:
+      - self-hosted
+      - Linux
+      - X64
+      - office
+      - android-qa
+    timeout-minutes: 90
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ env.BASE_BRANCH }}
+          clean: true
+          fetch-depth: 0
+          submodules: recursive
+
+      - name: Set up Java 21
+        uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: "21"
+          cache: gradle
+
+      - name: Validate Gradle wrapper
+        uses: gradle/actions/wrapper-validation@v6
+
+      - name: Set up Android SDK
+        uses: android-actions/setup-android@v4
+
+      - name: Install 1Password CLI
+        uses: 1password/install-cli-action@v3
+
+      - name: Fetch runtime secrets
+        run: OP_SERVICE_ACCOUNT_TOKEN="${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}" bash scripts/qa_android_ui_tests/execution_setup.sh fetch-runtime-secrets
+
+      # Baseline profile generation doesn't need production signing.
+      # The test APK is installed via adb and never distributed.
+      # ENABLE_SIGNING=FALSE makes both release and compatrelease use debug signing.
+      - name: Disable production signing
+        run: echo "ENABLE_SIGNING=FALSE" >> "$GITHUB_ENV"
+
+      - name: Verify at least one device connected
+        run: |
+          adb devices
+          DEVICE_COUNT="$(adb devices | awk '$2 == "device" { count++ } END { print count + 0 }')"
+          test "$DEVICE_COUNT" -ge 1
+
+      - name: Capture device metadata
+        id: device
+        run: |
+          echo "model=$(adb shell getprop ro.product.model | tr -d '\r')" >> "$GITHUB_OUTPUT"
+          echo "sdk=$(adb shell getprop ro.build.version.sdk | tr -d '\r')" >> "$GITHUB_OUTPUT"
+
+      - name: Generate baseline + startup profile
+        run: |
+          ./gradlew :app:generateProdCompatreleaseBaselineProfile \
+            -Pandroid.testInstrumentationRunnerArguments.class=com.wire.benchmark.BaselineGenerator \
+            -Pandroid.testInstrumentationRunnerArguments.BACKEND_NAME="STAGING" \
+            -Pandroid.testInstrumentationRunnerArguments.TARGET_PACKAGE="$TARGET_PACKAGE" \
+            --no-daemon \
+            --no-configuration-cache
+
+          mkdir -p app/src/main
+          cp app/src/prodCompatrelease/generated/baselineProfiles/baseline-prof.txt app/src/main/baseline-prof.txt
+          cp app/src/prodCompatrelease/generated/baselineProfiles/startup-prof.txt app/src/main/startup-prof.txt
+
+      - name: Verify generated profiles
+        run: |
+          test -s app/src/main/baseline-prof.txt
+          test -s app/src/main/startup-prof.txt
+
+      - name: Upload generated profiles
+        uses: actions/upload-artifact@v7
+        with:
+          name: baseline-profile-${{ github.run_id }}
+          path: |
+            app/src/prodCompatrelease/generated/baselineProfiles/*
+          if-no-files-found: error
+
+      - name: Create or update baseline profile PR
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.ANDROID_BOB_GH_TOKEN }}
+          branch: ${{ env.REFRESH_BRANCH }}
+          base: ${{ env.BASE_BRANCH }}
+          title: "chore(perf): refresh baseline and startup profiles (WPB-8645)"
+          commit-message: "chore: refresh baseline profile"
+          body: |
+            Auto-generated baseline profile refresh.
+
+            Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          labels: |
+            performance
+          add-paths: |
+            app/src/main/baseline-prof.txt
+            app/src/main/startup-prof.txt
+          delete-branch: false

benchmark/README.md

@@ -19,29 +19,49 @@ Run the startup benchmark without login:
   -Pandroid.testInstrumentationRunnerArguments.TARGET_PACKAGE="com.wire"
 ```
 
-Run the startup benchmark with login:
+Run the fixture-backed startup benchmark with login:
 
 ```shell
 ./gradlew :benchmark:connectedProdBenchmarkBenchmarkAndroidTest \
   -Pandroid.testInstrumentationRunnerArguments.class=com.wire.benchmark.StartupBenchmarkWithLogin#startUpWithoutBaselineProfiler \
   -Pandroid.testInstrumentationRunnerArguments.TARGET_PACKAGE="com.wire" \
-  -Pandroid.testInstrumentationRunnerArguments.EMAIL="$EMAIL" \
-  -Pandroid.testInstrumentationRunnerArguments.PASSWORD="$PASSWORD"
+  -Pandroid.testInstrumentationRunnerArguments.BACKEND_NAME="STAGING"
 ```
 
-Run the baseline profile generator:
+Run the fixture-backed baseline profile generator:
 
 ```shell
 ./gradlew :benchmark:connectedProdBenchmarkBenchmarkAndroidTest \
   -Pandroid.testInstrumentationRunnerArguments.class=com.wire.benchmark.BaselineGenerator \
   -Pandroid.testInstrumentationRunnerArguments.TARGET_PACKAGE="com.wire" \
+  -Pandroid.testInstrumentationRunnerArguments.BACKEND_NAME="STAGING"
+```
+
+Run the manual-credentials startup benchmark with login:
+
+```shell
+./gradlew :benchmark:connectedProdBenchmarkBenchmarkAndroidTest \
+  -Pandroid.testInstrumentationRunnerArguments.class=com.wire.benchmark.ManualStartupBenchmarkWithLogin#startUpWithoutBaselineProfiler \
+  -Pandroid.testInstrumentationRunnerArguments.TARGET_PACKAGE="com.wire" \
   -Pandroid.testInstrumentationRunnerArguments.EMAIL="$EMAIL" \
   -Pandroid.testInstrumentationRunnerArguments.PASSWORD="$PASSWORD"
 ```
 
+Run the manual-credentials baseline profile generator:
+
+```shell
+./gradlew :benchmark:connectedProdBenchmarkBenchmarkAndroidTest \
+  -Pandroid.testInstrumentationRunnerArguments.class=com.wire.benchmark.ManualBaselineGenerator \
+  -Pandroid.testInstrumentationRunnerArguments.TARGET_PACKAGE="com.wire" \
+  -Pandroid.testInstrumentationRunnerArguments.EMAIL="$EMAIL" \
+  -Pandroid.testInstrumentationRunnerArguments.PASSWORD="$PASSWORD" \
+  -Pandroid.testInstrumentationRunnerArguments.CONVERSATION_NAME="Some Conversation"
+```
+
 ## Notes
 
-- The benchmark module reads `TARGET_PACKAGE`, `EMAIL`, and `PASSWORD` from instrumentation runner arguments.
+- Fixture-backed classes use `TARGET_PACKAGE`, `BACKEND_NAME`, and optional `CONVERSATION_NAME`.
+- Manual classes use `TARGET_PACKAGE`, `EMAIL`, `PASSWORD`, and optional `CONVERSATION_NAME`.
 - The app APK is produced from `:app` automatically through `targetProjectPath = ":app"`.
 - Benchmark output is written under `benchmark/build/outputs/connected_android_test_additional_output/`.
 - The committed pre-profile reference snapshot lives under `benchmark/baselines/`.

benchmark/build.gradle.kts

@@ -36,6 +36,8 @@ android {
     compileOptions {
         sourceCompatibility = JavaVersion.VERSION_17
         targetCompatibility = JavaVersion.VERSION_17
+        // Required because :tests:testsSupport depends on java.time APIs.
+        isCoreLibraryDesugaringEnabled = true
     }
 
     flavorDimensions += FlavorDimensions.DEFAULT
@@ -61,6 +63,12 @@ android {
             matchingFallbacks += listOf("release")
         }
     }
+
+    sourceSets {
+        getByName("main") {
+            kotlin.directories.add(project(":tests:testsSupport").file("src/main").path)
+        }
+    }
 }
 
 baselineProfile {
@@ -72,5 +80,11 @@ dependencies {
     implementation(libs.androidx.espresso.core)
     implementation(libs.androidx.test.uiAutomator)
     implementation(libs.androidx.benchmark.macro.junit4)
+    implementation(libs.datafaker)
+    implementation(libs.gson)
     implementation(libs.junit4)
+    implementation(libs.zxing.android.embedded)
+    implementation(libs.zxing.core)
+    implementation(project(":tests:testsSupport"))
+    coreLibraryDesugaring(libs.android.desugarJdkLibs)
 }

benchmark/src/main/java/com/wire/benchmark/BaselineGenerator.kt

@@ -20,6 +20,7 @@ package com.wire.benchmark
 import androidx.benchmark.macro.junit4.BaselineProfileRule
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import androidx.test.platform.app.InstrumentationRegistry
+import androidx.test.platform.app.InstrumentationRegistry.getInstrumentation
 import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -32,8 +33,7 @@ class BaselineGenerator {
 
     private val args get() = InstrumentationRegistry.getArguments()
     private val targetPackage get() = args.getString("TARGET_PACKAGE", "com.wire")
-    private val email get() = args.getString("EMAIL").orEmpty()
-    private val password get() = args.getString("PASSWORD").orEmpty()
+    private val backendName get() = args.getString("BACKEND_NAME", "STAGING")
     private val conversationName get() = args.getString("CONVERSATION_NAME").orEmpty()
 
     @Test
@@ -43,10 +43,18 @@ class BaselineGenerator {
     ) {
         pressHome()
         startActivityAndWait()
-        if (email.isNotEmpty() && password.isNotEmpty()) {
-            login(email, password)
+        val fixture = BenchmarkFixtureFactory.create(
+            backendName = backendName,
+            context = getInstrumentation().context,
+            conversationNameOverride = conversationName,
+        )
+        try {
+            switchBackend(fixture.backend.deeplink)
+            login(fixture.email, fixture.password)
             openContactsAndReturn()
-            openConversation(conversationName)
+            openConversation(fixture.conversationName)
+        } finally {
+            fixture.cleanup()
         }
     }
 }

benchmark/src/main/java/com/wire/benchmark/BenchmarkFixture.kt

@@ -0,0 +1,98 @@
+/*
+ * Wire
+ * Copyright (C) 2026 Wire Swiss GmbH
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see http://www.gnu.org/licenses/.
+ */
+package com.wire.benchmark
+
+import android.content.Context
+import backendUtils.BackendClient
+import backendUtils.team.TeamHelper
+import backendUtils.team.TeamRoles
+import backendUtils.team.deleteTeam
+import backendUtils.user.removeBackendClients
+import service.TestServiceHelper
+import user.usermanager.ClientUserManager
+import java.util.UUID
+
+data class BenchmarkFixture(
+    val email: String,
+    val password: String,
+    val conversationName: String,
+    val backend: BackendClient,
+    private val usersManager: ClientUserManager,
+) {
+    fun cleanup() {
+        usersManager.createdUsers.forEach { user ->
+            runCatching { user.removeBackendClients(backend) }
+        }
+        usersManager.getAllTeamOwners().forEach { owner ->
+            runCatching { owner.deleteTeam(backend) }
+        }
+    }
+}
+
+object BenchmarkFixtureFactory {
+
+    @Suppress("MagicNumber")
+    fun create(
+        backendName: String,
+        context: Context,
+        conversationNameOverride: String = "",
+    ): BenchmarkFixture {
+        val backend = BackendClient.loadBackend(backendName)
+        val teamHelper = TeamHelper()
+        val testServiceHelper = TestServiceHelper(teamHelper.usersManager)
+        val suffix = UUID.randomUUID().toString().substring(0, 8)
+        val teamName = "BaselineProfile$suffix"
+        val conversationName = conversationNameOverride.ifEmpty { "BaselineConversation$suffix" }
+
+        teamHelper.usersManager.createTeamOwnerByAlias(
+            nameAlias = OWNER_ALIAS,
+            teamName = teamName,
+            locale = "en_US",
+            updateHandle = true,
+            backend = backend,
+            context = context,
+        )
+        teamHelper.userXAddsUsersToTeam(
+            ownerNameAlias = OWNER_ALIAS,
+            userNameAliases = MEMBER_ALIAS,
+            teamName = teamName,
+            role = TeamRoles.Member,
+            backendClient = backend,
+            context = context,
+            membersHaveHandles = true,
+        )
+        testServiceHelper.userHasGroupConversationInTeam(
+            chatOwnerNameAlias = OWNER_ALIAS,
+            chatName = conversationName,
+            otherParticipantsNameAlises = MEMBER_ALIAS,
+            teamName = teamName,
+        )
+
+        val owner = teamHelper.usersManager.findUserBy(OWNER_ALIAS, ClientUserManager.FindBy.NAME_ALIAS)
+        return BenchmarkFixture(
+            email = owner.email.orEmpty(),
+            password = owner.password.orEmpty(),
+            conversationName = conversationName,
+            backend = backend,
+            usersManager = teamHelper.usersManager,
+        )
+    }
+
+    private const val OWNER_ALIAS = "user1Name"
+    private const val MEMBER_ALIAS = "user2Name"
+}

benchmark/src/main/java/com/wire/benchmark/ConversationExt.kt

@@ -19,7 +19,6 @@ package com.wire.benchmark
 
 import androidx.benchmark.macro.MacrobenchmarkScope
 import androidx.test.uiautomator.By
-import androidx.test.uiautomator.UiObject2
 import androidx.test.uiautomator.Until
 import kotlin.time.Duration.Companion.seconds
 
@@ -31,23 +30,9 @@ fun MacrobenchmarkScope.openContactsAndReturn() {
 }
 
 fun MacrobenchmarkScope.openConversation(conversationName: String) {
-    val conversationTarget = if (conversationName.isNotEmpty()) {
-        device.findObject(By.text(conversationName))
-    } else {
-        firstVisibleConversationCandidate()
-    }
-
+    val conversationTarget = device.findObject(By.text(conversationName))
     conversationTarget?.click()
     device.wait(Until.hasObject(By.desc(" Type a message")), 10.seconds.inWholeMilliseconds)
     device.pressBack()
     device.wait(Until.hasObject(By.text("Conversations")), 10.seconds.inWholeMilliseconds)
 }
-
-private fun MacrobenchmarkScope.firstVisibleConversationCandidate(): UiObject2? {
-    val ignoredLabels = setOf("Conversations", "Contacts")
-    return device.findObjects(By.clazz("android.widget.TextView"))
-        .firstOrNull { candidate ->
-            val text = candidate.text?.trim().orEmpty()
-            text.isNotEmpty() && text !in ignoredLabels && candidate.visibleBounds.height() > 0
-        }
-}