#!/usr/bin/env python3 """Turn macrobenchmark's JSON into a table a human reads, and say when the jank metric is missing. ./tools/benchmark-report.py benchmark-results # after `adb pull` ./tools/benchmark-report.py --require-frame-duration results/ Macrobenchmark writes one `*-benchmarkData.json` per run into the *test* app's media directory: /sdcard/Android/media/de.softwareapp_hb.filemanager.benchmark/ and it has two kinds of number in it, which mean different things and are therefore kept apart: * `metrics` — one value per iteration. `frameCount` is how many frames an interaction took: a length, not a speed. * `sampledMetrics` — many values per iteration, summarised as percentiles. `frameDurationCpuMs` (how long each frame took) and `frameOverrunMs` (how far past its deadline it ran) are the jank numbers. This phone reports `frameCount` in the first group and, on some runs, `frameDurationCpuMs` in the second; whether the second group has the jank metric is exactly what a CI emulator run is for, so `--require-frame-duration` makes its absence a failure rather than a footnote. """ from __future__ import annotations import argparse import glob import json import os import sys def files_in(paths: list[str]) -> list[str]: """Every `*-benchmarkData.json` under the given files and directories, in a stable order.""" found: list[str] = [] for path in paths: if os.path.isdir(path): found += glob.glob(os.path.join(path, "**", "*.json"), recursive=True) else: found.append(path) return sorted(set(found)) def load(path: str) -> dict | None: """The run in [path], or `None` when the file is not macrobenchmark output.""" try: with open(path, encoding="utf-8") as handle: data = json.load(handle) except (OSError, json.JSONDecodeError): return None return data if isinstance(data, dict) and "benchmarks" in data else None def device_line(context: dict) -> str: build = context.get("build", {}) sdk = build.get("version", {}).get("sdk") name = " ".join(part for part in (build.get("brand"), build.get("model")) if part) fingerprint = build.get("fingerprint", "") # The fingerprint is the honest identifier: same model, different build, different numbers. return f"{name} (API {sdk}) — `{fingerprint}`" if fingerprint else f"{name} (API {sdk})" def number(value: float) -> str: """A value with as much precision as it needs: counts are whole, milliseconds are not.""" return str(int(value)) if float(value).is_integer() else f"{value:.1f}" def report(data: dict) -> tuple[list[str], bool]: """Markdown lines for one run, and whether it carried per-frame timings.""" lines: list[str] = [] context = data.get("context", {}) lines.append(f"**Device**: {device_line(context)}") lines.append(f"**Compilation**: `{context.get('compilationMode', 'unknown')}`") lines.append("") benchmarks = data.get("benchmarks", []) has_frame_duration = any("frameDurationCpuMs" in b.get("sampledMetrics", {}) for b in benchmarks) counted = sorted((b for b in benchmarks if b.get("metrics")), key=lambda b: b["name"]) if counted: # One value per iteration: `frameCount` is a length in frames, `timeToInitialDisplayMs` a # duration. They share a table because they are read the same way, but the unit is per row. lines.append("### One value per iteration") lines.append("") lines.append("| benchmark | metric | median | min | max | iterations |") lines.append("|---|---|---|---|---|---|") for benchmark in counted: for metric, stats in sorted(benchmark["metrics"].items()): runs = len(stats.get("runs", [])) unit = "frames" if metric == "frameCount" else "ms" lines.append( f"| `{benchmark['name']}` | {metric} | {number(stats['median'])} {unit} | " f"{number(stats['minimum'])} | {number(stats['maximum'])} | {runs} |", ) lines.append("") sampled = sorted((b for b in benchmarks if b.get("sampledMetrics")), key=lambda b: b["name"]) if sampled: lines.append("### Per-frame timing") lines.append("") lines.append("| benchmark | metric | P50 | P90 | P99 |") lines.append("|---|---|---|---|---|") for benchmark in sampled: for metric, stats in sorted(benchmark["sampledMetrics"].items()): lines.append( f"| `{benchmark['name']}` | {metric} | {number(stats['P50'])} ms | " f"{number(stats['P90'])} ms | {number(stats['P99'])} ms |", ) lines.append("") if not has_frame_duration: lines.append("> No `frameDurationCpuMs` in this run: the device reported frame counts only, so " "there is no jank number here — only how many frames each interaction took.") lines.append("") total = sum(b.get("totalRunTimeNs", 0) for b in benchmarks) / 1e9 plural = "benchmark" if len(benchmarks) == 1 else "benchmarks" lines.append(f"_{len(benchmarks)} {plural}, {total:.0f} s of measurement._") return lines, has_frame_duration def main() -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument( "paths", nargs="*", help="files or directories of macrobenchmark JSON (default: benchmark-results)", ) parser.add_argument( "--require-frame-duration", action="store_true", help="exit non-zero unless the run reported frameDurationCpuMs", ) arguments = parser.parse_args() paths = arguments.paths or ["benchmark-results"] runs = [(path, data) for path in files_in(paths) if (data := load(path)) is not None] if not runs: print(f"benchmark-report: no macrobenchmark JSON in {', '.join(paths)}", file=sys.stderr) return 2 anywhere = False for index, (path, data) in enumerate(runs): if len(runs) > 1: print(f"## {os.path.basename(path)}") print() lines, has_frame_duration = report(data) anywhere = anywhere or has_frame_duration print("\n".join(lines)) if index < len(runs) - 1: print() if arguments.require_frame_duration and not anywhere: print( "benchmark-report: frameDurationCpuMs is absent from every run, so no jank metric was " "measured", file=sys.stderr, ) return 1 return 0 if __name__ == "__main__": sys.exit(main())