SoftwareApp
66183a8596
Item 5 of the queue: .github/workflows/benchmark.yml measures on an API 35 emulator (reactivecircus/android-emulator-runner, google_apis, x86_64, SwiftShader), on demand and every Monday. Not on pull requests: a run is minutes of measurement behind twenty of booting an emulator, and the gate is meant to answer in one or two. It installs both APKs and instruments directly instead of using connectedBenchmarkAndroidTest, because that task installs the app itself - which undoes the compile reset that makes the measurement a never-run baseline. tools/benchmark-report.py turns the JSON into two tables, keeping frameCount (a length) apart from frameDurationCpuMs and frameOverrunMs (speeds), and --require-frame-duration fails the job when the second table would be empty: the metric is the reason the job exists. Driving the workflow's own script against the phone first paid for itself three times. It found that cmd package compile --reset needs adbd as root - a user build refuses it with SecurityException, so the workflow now does adb root first (google_apis allows it) instead of silently measuring a compiled app. It produced the second run of the 20 000-file open (16 frames median, 32 max) and the first numbers for scroll, search and cold start at that size: search costs 319 frames, which is now the biggest number in the file. And it showed the phone reporting frameDurationCpuMs and frameOverrunMs after all, so PERFORMANCE.md's 'this phone reports counts only' was a property of those earlier runs, not of the handset - corrected, with the numbers.
164 lines
6.5 KiB
Python
Executable File
164 lines
6.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Turn macrobenchmark's JSON into a table a human reads, and say when the jank metric is missing.
|
|
|
|
./tools/benchmark-report.py benchmark-results # after `adb pull`
|
|
./tools/benchmark-report.py --require-frame-duration results/
|
|
|
|
Macrobenchmark writes one `*-benchmarkData.json` per run into the *test* app's media directory:
|
|
|
|
/sdcard/Android/media/de.softwareapp_hb.filemanager.benchmark/
|
|
|
|
and it has two kinds of number in it, which mean different things and are therefore kept apart:
|
|
|
|
* `metrics` — one value per iteration. `frameCount` is how many frames an interaction took: a length,
|
|
not a speed.
|
|
* `sampledMetrics` — many values per iteration, summarised as percentiles. `frameDurationCpuMs` (how
|
|
long each frame took) and `frameOverrunMs` (how far past its deadline it ran) are the jank numbers.
|
|
|
|
This phone reports `frameCount` in the first group and, on some runs, `frameDurationCpuMs` in the
|
|
second; whether the second group has the jank metric is exactly what a CI emulator run is for, so
|
|
`--require-frame-duration` makes its absence a failure rather than a footnote.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import glob
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
|
|
def files_in(paths: list[str]) -> list[str]:
|
|
"""Every `*-benchmarkData.json` under the given files and directories, in a stable order."""
|
|
found: list[str] = []
|
|
for path in paths:
|
|
if os.path.isdir(path):
|
|
found += glob.glob(os.path.join(path, "**", "*.json"), recursive=True)
|
|
else:
|
|
found.append(path)
|
|
return sorted(set(found))
|
|
|
|
|
|
def load(path: str) -> dict | None:
|
|
"""The run in [path], or `None` when the file is not macrobenchmark output."""
|
|
try:
|
|
with open(path, encoding="utf-8") as handle:
|
|
data = json.load(handle)
|
|
except (OSError, json.JSONDecodeError):
|
|
return None
|
|
return data if isinstance(data, dict) and "benchmarks" in data else None
|
|
|
|
|
|
def device_line(context: dict) -> str:
|
|
build = context.get("build", {})
|
|
sdk = build.get("version", {}).get("sdk")
|
|
name = " ".join(part for part in (build.get("brand"), build.get("model")) if part)
|
|
fingerprint = build.get("fingerprint", "")
|
|
# The fingerprint is the honest identifier: same model, different build, different numbers.
|
|
return f"{name} (API {sdk}) — `{fingerprint}`" if fingerprint else f"{name} (API {sdk})"
|
|
|
|
|
|
def number(value: float) -> str:
|
|
"""A value with as much precision as it needs: counts are whole, milliseconds are not."""
|
|
return str(int(value)) if float(value).is_integer() else f"{value:.1f}"
|
|
|
|
|
|
def report(data: dict) -> tuple[list[str], bool]:
|
|
"""Markdown lines for one run, and whether it carried per-frame timings."""
|
|
lines: list[str] = []
|
|
context = data.get("context", {})
|
|
lines.append(f"**Device**: {device_line(context)}")
|
|
lines.append(f"**Compilation**: `{context.get('compilationMode', 'unknown')}`")
|
|
lines.append("")
|
|
|
|
benchmarks = data.get("benchmarks", [])
|
|
has_frame_duration = any("frameDurationCpuMs" in b.get("sampledMetrics", {}) for b in benchmarks)
|
|
|
|
counted = sorted((b for b in benchmarks if b.get("metrics")), key=lambda b: b["name"])
|
|
if counted:
|
|
# One value per iteration: `frameCount` is a length in frames, `timeToInitialDisplayMs` a
|
|
# duration. They share a table because they are read the same way, but the unit is per row.
|
|
lines.append("### One value per iteration")
|
|
lines.append("")
|
|
lines.append("| benchmark | metric | median | min | max | iterations |")
|
|
lines.append("|---|---|---|---|---|---|")
|
|
for benchmark in counted:
|
|
for metric, stats in sorted(benchmark["metrics"].items()):
|
|
runs = len(stats.get("runs", []))
|
|
unit = "frames" if metric == "frameCount" else "ms"
|
|
lines.append(
|
|
f"| `{benchmark['name']}` | {metric} | {number(stats['median'])} {unit} | "
|
|
f"{number(stats['minimum'])} | {number(stats['maximum'])} | {runs} |",
|
|
)
|
|
lines.append("")
|
|
|
|
sampled = sorted((b for b in benchmarks if b.get("sampledMetrics")), key=lambda b: b["name"])
|
|
if sampled:
|
|
lines.append("### Per-frame timing")
|
|
lines.append("")
|
|
lines.append("| benchmark | metric | P50 | P90 | P99 |")
|
|
lines.append("|---|---|---|---|---|")
|
|
for benchmark in sampled:
|
|
for metric, stats in sorted(benchmark["sampledMetrics"].items()):
|
|
lines.append(
|
|
f"| `{benchmark['name']}` | {metric} | {number(stats['P50'])} ms | "
|
|
f"{number(stats['P90'])} ms | {number(stats['P99'])} ms |",
|
|
)
|
|
lines.append("")
|
|
|
|
if not has_frame_duration:
|
|
lines.append("> No `frameDurationCpuMs` in this run: the device reported frame counts only, so "
|
|
"there is no jank number here — only how many frames each interaction took.")
|
|
lines.append("")
|
|
|
|
total = sum(b.get("totalRunTimeNs", 0) for b in benchmarks) / 1e9
|
|
plural = "benchmark" if len(benchmarks) == 1 else "benchmarks"
|
|
lines.append(f"_{len(benchmarks)} {plural}, {total:.0f} s of measurement._")
|
|
return lines, has_frame_duration
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
parser.add_argument(
|
|
"paths",
|
|
nargs="*",
|
|
help="files or directories of macrobenchmark JSON (default: benchmark-results)",
|
|
)
|
|
parser.add_argument(
|
|
"--require-frame-duration",
|
|
action="store_true",
|
|
help="exit non-zero unless the run reported frameDurationCpuMs",
|
|
)
|
|
arguments = parser.parse_args()
|
|
paths = arguments.paths or ["benchmark-results"]
|
|
|
|
runs = [(path, data) for path in files_in(paths) if (data := load(path)) is not None]
|
|
if not runs:
|
|
print(f"benchmark-report: no macrobenchmark JSON in {', '.join(paths)}", file=sys.stderr)
|
|
return 2
|
|
|
|
anywhere = False
|
|
for index, (path, data) in enumerate(runs):
|
|
if len(runs) > 1:
|
|
print(f"## {os.path.basename(path)}")
|
|
print()
|
|
lines, has_frame_duration = report(data)
|
|
anywhere = anywhere or has_frame_duration
|
|
print("\n".join(lines))
|
|
if index < len(runs) - 1:
|
|
print()
|
|
|
|
if arguments.require_frame_duration and not anywhere:
|
|
print(
|
|
"benchmark-report: frameDurationCpuMs is absent from every run, so no jank metric was "
|
|
"measured",
|
|
file=sys.stderr,
|
|
)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|