Get a real cold-start number out of the benchmark, and record what the device demanded
coldStart measures 532 ms median (min 499, max 805) on the benchmark build - R8, not debuggable, profileable, CompilationMode.None - against 1814 ms on the debug build, which is the honest illustration of why a debug number must never be published as the app's performance. The tool also warns that it cannot clear the runtime image on this Android version, so the figure is optimistic by an unmeasured amount; that is in PERFORMANCE.md next to the number rather than in a footnote. The three interaction benchmarks are written and compile but have not produced a number, and getting that far took four fixes worth naming. The app under test cannot keep the app's package id (signature) and cannot use .benchmark (the test APK owns that id, INSTALL_FAILED_VERSION_DOWNGRADE); macrobenchmark needs package visibility for it or it reports NameNotFoundException for an app that pm list packages shows installed; UiAutomation.executeShellCommand runs as the test app's uid, so the fixture has to be made through adb instead - that one cost the first attempt a hang; and testBuildType has to be behind a flag or :app:testDebugUnitTest stops existing. The workflow that works on a device with an install gate is now documented: install the two APKs by hand once, run tools/benchmark-fixture.sh, then am instrument. The Gradle task is documented as the alternative and as the reason the app under test disappeared once - it uninstalls around the run. Gate green, map current, 386 tests unchanged.
This commit is contained in:
@@ -1,3 +1,22 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- The benchmark APK needs no manifest entries of its own; see build.gradle.kts. -->
|
||||
<manifest />
|
||||
<!--
|
||||
What the benchmark APK needs to see the app it measures.
|
||||
|
||||
Android 11+ hides other packages from an app that does not ask for them, and macrobenchmark looks the
|
||||
target up with PackageManager before doing anything else. Without visibility it fails with
|
||||
`NameNotFoundException: …filemanager.perf` — even though `adb shell pm list packages` shows the app
|
||||
installed, which is what made this look like a missing install rather than a visibility rule.
|
||||
|
||||
`<queries>` alone was not enough on this device, so the test APK also asks for QUERY_ALL_PACKAGES. That
|
||||
permission is in no shipped artefact: this manifest belongs to the benchmark module, which never goes
|
||||
to a store.
|
||||
-->
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
xmlns:tools="http://schemas.android.com/tools">
|
||||
<uses-permission
|
||||
android:name="android.permission.QUERY_ALL_PACKAGES"
|
||||
tools:ignore="QueryAllPackagesPermission" />
|
||||
<queries>
|
||||
<package android:name="de.softwareapp_hb.filemanager.perf" />
|
||||
</queries>
|
||||
</manifest>
|
||||
|
||||
+13
-42
@@ -7,11 +7,9 @@ import androidx.benchmark.macro.StartupMode
|
||||
import androidx.benchmark.macro.StartupTimingMetric
|
||||
import androidx.benchmark.macro.junit4.MacrobenchmarkRule
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4
|
||||
import androidx.test.platform.app.InstrumentationRegistry
|
||||
import androidx.test.uiautomator.By
|
||||
import androidx.test.uiautomator.UiDevice
|
||||
import androidx.test.uiautomator.Until
|
||||
import org.junit.AfterClass
|
||||
import org.junit.Assert.assertNotNull
|
||||
import org.junit.Rule
|
||||
import org.junit.Test
|
||||
@@ -50,6 +48,12 @@ class BrowserBenchmark {
|
||||
*/
|
||||
private val app = "de.softwareapp_hb.filemanager.perf"
|
||||
|
||||
// No setup by shell command here on purpose. UiAutomation's executeShellCommand runs with *this
|
||||
// app's* uid, not the shell user's, so it cannot create files in /sdcard and cannot grant itself
|
||||
// an app-op — the first attempt failed exactly that way, with a RuntimeException from
|
||||
// UiAutomationConnection and no fixture on disk. `tools/benchmark-fixture.sh` prepares the device
|
||||
// instead: it runs through adb, so it has the shell's rights.
|
||||
|
||||
@Test
|
||||
fun coldStart() = benchmark.measureRepeated(
|
||||
packageName = app,
|
||||
@@ -119,11 +123,11 @@ class BrowserBenchmark {
|
||||
// ------------------------------------------------------------------ setup
|
||||
|
||||
/**
|
||||
* Everything the measured block must not pay for: the access grant, the fixture on disk, and the
|
||||
* app open on the volume list with the fixture's row on screen.
|
||||
* The device is expected to be prepared already: `tools/benchmark-fixture.sh` creates the fixture
|
||||
* and grants the app its storage access. This only navigates to it, and fails with that
|
||||
* instruction if the row is not where it should be.
|
||||
*/
|
||||
private fun MacrobenchmarkScope.openFixture(device: UiDevice) {
|
||||
Fixture.ensure(device, app)
|
||||
pressHome()
|
||||
startActivityAndWait()
|
||||
val volume = device.wait(Until.findObject(By.text("Internal storage")), TIMEOUT_MS)
|
||||
@@ -134,7 +138,8 @@ class BrowserBenchmark {
|
||||
)
|
||||
volume!!.click()
|
||||
assertNotNull(
|
||||
"the fixture folder is not in the volume root, so setup did not complete",
|
||||
"the fixture folder is not in the volume root. Run tools/benchmark-fixture.sh first — " +
|
||||
"it creates /sdcard/${Fixture.NAME} on the device and grants the app storage access.",
|
||||
device.wait(Until.findObject(By.text(Fixture.NAME)), TIMEOUT_MS),
|
||||
)
|
||||
}
|
||||
@@ -144,21 +149,12 @@ class BrowserBenchmark {
|
||||
|
||||
/** The app bar's own description for the search action; see strings.xml. */
|
||||
private const val SEARCH_ACTION = "Search this folder"
|
||||
|
||||
/**
|
||||
* Removes the fixture once the class is done. A benchmark that leaves a thousand files in
|
||||
* someone's storage is a benchmark that stops being run — and this runs on real phones.
|
||||
*/
|
||||
@JvmStatic
|
||||
@AfterClass
|
||||
fun cleanUp() {
|
||||
Fixture.remove(UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The folder the interaction benchmarks use: a thousand empty files, created once and reused.
|
||||
* The folder the interaction benchmarks use: a thousand empty files, created and removed by
|
||||
* `tools/benchmark-fixture.sh` rather than by the test — see the note on the class.
|
||||
*
|
||||
* It sits at the root of shared storage under a name that sorts before everything else, because the
|
||||
* alternative — scrolling until the fixture appears — is another interaction to keep working as the
|
||||
@@ -167,30 +163,5 @@ class BrowserBenchmark {
|
||||
private object Fixture {
|
||||
const val NAME = "AABenchmark"
|
||||
const val FIRST_FILE = "file-0.txt"
|
||||
private const val COUNT = 1_000
|
||||
private const val DIRECTORY = "/sdcard/$NAME"
|
||||
private const val MARKER = "$DIRECTORY/.ready-$COUNT"
|
||||
|
||||
fun ensure(device: UiDevice, app: String) {
|
||||
// The benchmark build is not the debug build, so it starts without the access grant. Granting
|
||||
// it through appops is what makes an unattended run possible: the alternative is a permission
|
||||
// dialog no test can answer.
|
||||
shell(device, "appops set --uid $app MANAGE_EXTERNAL_STORAGE allow")
|
||||
|
||||
if (shell(device, "test -f $MARKER && echo ready || echo missing") == "ready") return
|
||||
|
||||
shell(device, "rm -rf $DIRECTORY; mkdir -p $DIRECTORY")
|
||||
// One shell loop rather than a thousand pushes: `: > file` is a builtin redirection, so this
|
||||
// costs a single shell and no fork per file.
|
||||
shell(device, "i=0; while [ \$i -lt $COUNT ]; do : > $DIRECTORY/file-\$i.txt; i=\$((i+1)); done")
|
||||
shell(device, "touch $MARKER")
|
||||
}
|
||||
|
||||
fun remove(device: UiDevice) {
|
||||
shell(device, "rm -rf $DIRECTORY")
|
||||
}
|
||||
|
||||
private fun shell(device: UiDevice, command: String): String =
|
||||
device.executeShellCommand(command).trim()
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user