Add a macrobenchmark module, and find out what the build was hiding
docs/PERFORMANCE.md has been honest that three of its four numbers do not exist. They exist now as measurements, if not yet as results: benchmark/ is a macrobenchmark module with four tests - cold start with StartupTimingMetric, then opening, scrolling and searching a folder of a thousand files with FrameTimingMetric - sharing a fixture at /sdcard/AABenchmark that is created once per run and deleted in @AfterClass. Frames rather than a named trace section on purpose: what a user calls slow is jank on screen, and the alternative puts benchmarking calls into production code for the sake of a number. CompilationMode.None() is the honest cold-install baseline; Partial() plus a baseline profile is the follow-up and needs the same module. The storage grant is applied with appops in setup, which is what lets the run be unattended - a permission dialog is the one thing a test cannot answer. Three build facts had to be discovered rather than designed, and each is now written where it will be read. The app under test cannot keep its own package id: it needs a debug signature to install on a test device, so it cannot replace a release install, and an un-suffixed build is exactly that. It also cannot use the obvious suffix: `.benchmark` is already the benchmark module's own APK id, and the test APK then fails to install with INSTALL_FAILED_VERSION_DOWNGRADE - which is what the first attempt did. Hence `.perf`. And `testBuildType = "benchmark"` cannot be set unconditionally, because it makes benchmark the variant that *unit* tests run against and :app:testDebugUnitTest - the task the entire gate is built on - stops existing; it is behind -PbenchmarkBuild instead. All three were measured, not guessed, and the third would have broken CI silently. Two smaller things came with it. tools/verify.sh --release now also compiles the app's benchmark variant and the test APK, because build wiring nobody runs is wiring that rots; no device is involved. And the code map grew an EXTRA_ROOTS section so it indexes benchmark/ too - a map that silently omits a module lies by omission, which is worse than the one line it costs. The numbers themselves are still missing, and for a device reason rather than a code one: installing the new package id on this MIUI phone fails with INSTALL_FAILED_USER_RESTRICTED (the "Install via USB" gate, which wants a human and answered too late). Both APKs are pushed to /sdcard/Download/ for a manual install, and the run is one command away: `adb shell am instrument -w -e class de.softwareapp_hb.filemanager.benchmark.BrowserBenchmark de.softwareapp_hb.filemanager.benchmark/androidx.test.runner.AndroidJUnitRunner`. Gate green with --release, including the new compile step; map current; 386 tests unchanged.
This commit is contained in:
+29
-1
@@ -49,6 +49,16 @@ PACKAGES = {
|
||||
"": "Application entry points",
|
||||
}
|
||||
|
||||
# Source outside app/src/main. The benchmark module is source too, and a map that silently omits a
|
||||
# module is a map that lies by omission. (name in the map, path from the repository root, blurb)
|
||||
EXTRA_ROOTS = [
|
||||
(
|
||||
"benchmark/",
|
||||
"benchmark/src/main/java/de/softwareapp_hb/filemanager/benchmark",
|
||||
"The macrobenchmark test APK: four measurements over a fixture folder, run on a device",
|
||||
),
|
||||
]
|
||||
|
||||
DECLARATION = re.compile(
|
||||
r"^(?:(?:public|internal|private|protected|abstract|open|sealed|data|value|inline|"
|
||||
r"suspend|operator|override|const|lateinit|external|annotation|enum)\s+)*"
|
||||
@@ -187,6 +197,8 @@ def render(root: Path) -> str:
|
||||
test_dir = root / "app/src/test/java" / PACKAGE
|
||||
sources = scan_source(main_dir)
|
||||
tests = scan_tests(test_dir)
|
||||
extras = [(name, blurb, scan_source(root / path)) for name, path, blurb in EXTRA_ROOTS]
|
||||
extra_files = [entry for _, _, entries in extras for entry in entries]
|
||||
|
||||
by_declaration = {}
|
||||
by_stem = {}
|
||||
@@ -218,7 +230,8 @@ def render(root: Path) -> str:
|
||||
"%d source files, %d lines - %d test files, %d tests. The behaviour index at the end "
|
||||
"lists every test name, and in this project a test name is a sentence about what the "
|
||||
"app does, so it doubles as the specification.\n"
|
||||
% (len(sources), sum(entry["lines"] for entry in sources),
|
||||
% (len(sources) + len(extra_files),
|
||||
sum(entry["lines"] for entry in sources) + sum(entry["lines"] for entry in extra_files),
|
||||
len(tests), sum(len(test["tests"]) for test in tests)),
|
||||
"Start at `AGENTS.md`. What is done, and why: `docs/ROADMAP.md` and `CHANGELOG.md`. "
|
||||
"What was verified by hand rather than by a test: `docs/EDGE_CASE_TESTING.md`.\n",
|
||||
@@ -245,6 +258,21 @@ def render(root: Path) -> str:
|
||||
% (Path(entry["rel"]).name, entry["lines"],
|
||||
cell.replace("|", "\\|").strip(), covered))
|
||||
|
||||
for label, blurb, entries in extras:
|
||||
if not entries:
|
||||
continue
|
||||
out.append("\n## %s\n" % label)
|
||||
out.append("%s\n" % blurb)
|
||||
out.append("\n| file | lines | what it holds | tests |")
|
||||
out.append("|---|---|---|---|")
|
||||
for entry in entries:
|
||||
declared = ", ".join(name + "()" if kind == "fun" else name
|
||||
for kind, name, doc, line in entry["decls"])
|
||||
cell = "%s%s" % ("%s " % entry["doc"] if entry["doc"] else "", declared)
|
||||
out.append("| `%s` | %d | %s | - |"
|
||||
% (Path(entry["rel"]).name, entry["lines"],
|
||||
cell.replace("|", "\\|").strip()))
|
||||
|
||||
out.append("\n## Behaviour index\n")
|
||||
out.append("Every test name in the suite, by file: the fastest way to see whether "
|
||||
"something is already covered, and what the app claims to do.\n")
|
||||
|
||||
@@ -45,6 +45,11 @@ if $release; then
|
||||
# runs, and a release-only defect (a class reached only through a resource, for instance) is
|
||||
# exactly what this step is for.
|
||||
gradle :app:assembleRelease
|
||||
|
||||
step "Benchmark APKs compile"
|
||||
# No device: this only proves that :benchmark still compiles against the app's benchmark build
|
||||
# type, which is the thing that rots silently between device runs.
|
||||
gradle :app:assembleBenchmark :benchmark:assembleDebug
|
||||
fi
|
||||
|
||||
printf '\n\033[1mverify: all checks passed\033[0m\n'
|
||||
|
||||
Reference in New Issue
Block a user