file-manager/tools/make-code-map.py

#!/usr/bin/env python3
"""
Builds docs/CODE_MAP.md: the one-file index of what lives where.

    python3 tools/make-code-map.py           # write docs/CODE_MAP.md
    python3 tools/make-code-map.py --check   # exit 1 if the file on disk is stale

Why this exists: the source is 12 000 lines across 74 files, which is more than fits in
anyone's head - and the first question about any change is "which file, and what does it
do?". A hand-written overview of that would be out of date within a milestone, so this one
is *derived*: every line comes from a declaration or from the KDoc already next to it, and
`--check` fails the moment the file stops matching the tree.

The rule that makes the map useful rather than merely long: a file is described by the KDoc
of the declaration that shares its name (`BatchRename.kt` -> `BatchRename`), which is the
convention this codebase already follows. When no declaration matches, the first KDoc in the
file is used, and the file is listed at the end so the convention can be checked rather than
assumed.

Test names are sentences about behaviour in this project, so the behaviour index at the end
doubles as the specification: every name is a claim about what the app does.
"""
import argparse
import re
import sys
from pathlib import Path

PACKAGE = "de/softwareapp_hb/filemanager"

# One line per package, because a directory has no comment of its own. Kept here rather than
# in the generated file so there is exactly one place to update it; the README's layout
# section says the same thing more briefly.
PACKAGES = {
    "core": "Pure Kotlin, **no Android imports**: paths, file names, kinds, sorting, "
            "formatting, matching, the batch-rename planner",
    "data": "Storage backends (all-files access and SAF), volumes, incoming shares, the "
            "SQLite search index and trash index",
    "domain": "Operations over a source: copy/move/delete/rename, archives, search walking, "
              "duplicate finding, trash, usage analysis",
    "service": "The foreground service that keeps a long operation alive and reports it",
    "text": "Domain enum to string-resource mapping",
    "ui/browser": "The browser screen: view model, state, list and grid, dialogs, selection bar",
    "ui/viewer": "The built-in viewers (text, image, PDF, media, markup, archive contents)",
    "ui/usage": "The storage-usage screen",
    "ui/trash": "The trash screen",
    "ui/duplicates": "The duplicate-finder screen",
    "ui/theme": "Compose theme, colours, type",
    "ui": "Compose screens and their shared pieces",
    "": "Application entry points",
}

DECLARATION = re.compile(
    r"^(?:(?:public|internal|private|protected|abstract|open|sealed|data|value|inline|"
    r"suspend|operator|override|const|lateinit|external|annotation|enum)\s+)*"
    r"(enum class|sealed class|sealed interface|data class|data object|annotation class|"
    r"value class|class|object|interface|fun|val|var|typealias)\s+([A-Za-z_][A-Za-z0-9_]*)"
)
DOC_START = re.compile(r"^\s*/\*\*")
TEST_FUN = re.compile(r"^\s*fun\s+`([^`]+)`")
TEST_CLASS = re.compile(r"^(?:class|object)\s+([A-Za-z_][A-Za-z0-9_]*)")


def repo_root() -> Path:
    """The repository root, found from this file rather than from the working directory."""
    here = Path(__file__).resolve()
    for candidate in (here.parent, *here.parents):
        if (candidate / "settings.gradle.kts").is_file():
            return candidate
    sys.exit("make-code-map.py: no settings.gradle.kts above %s" % here)


def first_sentence(block: str, limit: int = 190) -> str:
    """The first sentence of a KDoc block, as one line and without markdown decoration."""
    parts = []
    for line in block.splitlines():
        line = line.strip().lstrip("/").lstrip("*").strip()
        if line.startswith("@") or not line:
            continue
        parts.append(line)
    joined = " ".join(parts)
    joined = re.sub(r"\[([^\]]+)]", r"\1", joined)
    joined = re.sub(r"`([^`]*)`", r"\1", joined)
    joined = re.sub(r"\*+", "", joined)
    match = re.search(r"^(.*?[.!?])(\s|$)", joined)
    sentence = (match.group(1) if match else joined).strip()
    if len(sentence) > limit:
        sentence = sentence[:limit].rsplit(" ", 1)[0] + "…"
    return sentence


def kdocs(source: str):
    """Every KDoc block: (start line, body, line of the declaration it documents)."""
    lines = source.splitlines()
    blocks = []
    index = 0
    while index < len(lines):
        if DOC_START.match(lines[index]):
            end = index
            body = []
            while end < len(lines):
                body.append(lines[end])
                if "*/" in lines[end]:
                    break
                end += 1
            after = end + 1
            while after < len(lines) and (not lines[after].strip()
                                          or lines[after].lstrip().startswith("@")):
                after += 1
            blocks.append((index, "\n".join(body), after))
            index = end + 1
        else:
            index += 1
    return blocks


def declarations(source: str):
    """Top-level declarations, as (kind, name, doc first sentence, line number)."""
    lines = source.splitlines()
    blocks = kdocs(source)
    doc_of = {after: first_sentence(body) for start, body, after in blocks}
    found = []
    for index, line in enumerate(lines):
        if not line or line[0].isspace():
            continue
        match = DECLARATION.match(line)
        if not match:
            continue
        found.append((match.group(1), match.group(2), doc_of.get(index), index + 1))
    return found


def purpose_of(path: Path, decls):
    """The KDoc of the declaration sharing the file's name, else the first one there is."""
    for kind, name, doc, line in decls:
        if name == path.stem and doc:
            return doc, True
    for kind, name, doc, line in decls:
        if doc:
            return doc, False
    return "", False


def scan_source(main_dir: Path):
    entries = []
    for path in sorted(main_dir.rglob("*.kt")):
        source = path.read_text(encoding="utf-8")
        decls = declarations(source)
        doc, matched = purpose_of(path, decls)
        entries.append({
            "rel": path.relative_to(main_dir).as_posix(),
            "package": path.parent.relative_to(main_dir).as_posix(),
            "stem": path.stem,
            "lines": len(source.splitlines()),
            "decls": decls,
            "doc": doc,
            "matched": matched,
        })
    return entries


def scan_tests(test_dir: Path):
    entries = []
    for path in sorted(test_dir.rglob("*.kt")):
        source = path.read_text(encoding="utf-8")
        lines = source.splitlines()
        names = [TEST_FUN.match(line).group(1) for line in lines if TEST_FUN.match(line)]
        if not names:
            continue
        subjects = [match.group(1)[:-4]
                    for match in (TEST_CLASS.match(line) for line in lines)
                    if match and match.group(1).endswith("Test")]
        # A test class is the usual way to say what is covered; a test file named after the file
        # rather than after a declaration (FileNamesTest -> FileNames.kt) is the other, so both
        # are offered and the source tree decides which one lands.
        subjects.append(path.stem[:-4] if path.stem.endswith("Test") else path.stem)
        entries.append({
            "rel": path.relative_to(test_dir).as_posix(),
            "stem": path.stem,
            "subjects": subjects,
            "tests": names,
        })
    return entries


def render(root: Path) -> str:
    main_dir = root / "app/src/main/java" / PACKAGE
    test_dir = root / "app/src/test/java" / PACKAGE
    sources = scan_source(main_dir)
    tests = scan_tests(test_dir)

    by_declaration = {}
    by_stem = {}
    for entry in sources:
        for kind, name, doc, line in entry["decls"]:
            by_declaration.setdefault(name, entry)
        by_stem.setdefault(entry["stem"], entry)

    tests_for = {}
    unplaced = []
    for test in tests:
        placed = False
        for subject in test["subjects"]:
            target = by_declaration.get(subject) or by_stem.get(subject)
            if target is None:
                continue
            bucket = tests_for.setdefault(target["rel"], [])
            if test not in bucket:
                bucket.append(test)
            placed = True
        if not placed:
            unplaced.append(test)

    out = [
        "# Code map\n",
        "Generated from the source by `tools/make-code-map.py` - do not edit by hand. "
        "Regenerate it after a change (`python3 tools/make-code-map.py`); `--check` exits "
        "non-zero when this file is stale.\n",
        "%d source files, %d lines - %d test files, %d tests. The behaviour index at the end "
        "lists every test name, and in this project a test name is a sentence about what the "
        "app does, so it doubles as the specification.\n"
        % (len(sources), sum(entry["lines"] for entry in sources),
           len(tests), sum(len(test["tests"]) for test in tests)),
        "Start at `AGENTS.md`. What is done, and why: `docs/ROADMAP.md` and `CHANGELOG.md`. "
        "What was verified by hand rather than by a test: `docs/EDGE_CASE_TESTING.md`.\n",
    ]

    packages = list(PACKAGES) + [name for name in sorted({entry["package"] for entry in sources})
                                 if name not in PACKAGES]
    for package in packages:
        entries = [entry for entry in sources if entry["package"] == package]
        if not entries:
            continue
        out.append("\n## %s\n" % (package + "/" if package else "(root files)"))
        if PACKAGES.get(package):
            out.append("%s\n" % PACKAGES[package])
        out.append("\n| file | lines | what it holds | tests |")
        out.append("|---|---|---|---|")
        for entry in entries:
            declared = ", ".join(name + "()" if kind == "fun" else name
                                 for kind, name, doc, line in entry["decls"])
            cell = "%s%s" % ("%s " % entry["doc"] if entry["doc"] else "", declared)
            covered = ", ".join("`%s` (%d)" % (test["stem"], len(test["tests"]))
                                for test in tests_for.get(entry["rel"], [])) or "-"
            out.append("| `%s` | %d | %s | %s |"
                       % (Path(entry["rel"]).name, entry["lines"],
                          cell.replace("|", "\\|").strip(), covered))

    out.append("\n## Behaviour index\n")
    out.append("Every test name in the suite, by file: the fastest way to see whether "
               "something is already covered, and what the app claims to do.\n")
    for test in tests:
        out.append("\n### `%s` (%d)\n" % (test["rel"], len(test["tests"])))
        out.extend("- %s" % name for name in test["tests"])
    if unplaced:
        out.append("\n### Test files with no source file of the same name\n")
        for test in unplaced:
            out.append("- `%s` (%d tests) - %s"
                       % (test["rel"], len(test["tests"]),
                          ", ".join(test["subjects"]) or "no named subject"))

    fallbacks = [entry for entry in sources if not entry["matched"]]
    out.append("\n---\n")
    out.append("Purpose taken from the first KDoc rather than from a declaration sharing the "
               "file's name (%d of %d): %s\n"
               % (len(fallbacks), len(sources),
                  ", ".join("`%s`" % Path(entry["rel"]).name for entry in fallbacks) or "none"))
    return re.sub(r"\n{3,}", "\n\n", "\n".join(out)).rstrip("\n") + "\n"


def main() -> int:
    parser = argparse.ArgumentParser(description="Build or verify docs/CODE_MAP.md")
    parser.add_argument("--check", action="store_true",
                        help="exit non-zero if docs/CODE_MAP.md is out of date")
    args = parser.parse_args()

    root = repo_root()
    target = root / "docs/CODE_MAP.md"
    rendered = render(root)

    if args.check:
        current = target.read_text(encoding="utf-8") if target.is_file() else ""
        if current == rendered:
            print("docs/CODE_MAP.md is current")
            return 0
        current_lines = current.splitlines()
        rendered_lines = rendered.splitlines()
        for number in range(max(len(current_lines), len(rendered_lines))):
            old = current_lines[number] if number < len(current_lines) else "<missing>"
            new = rendered_lines[number] if number < len(rendered_lines) else "<missing>"
            if old != new:
                print("docs/CODE_MAP.md is out of date at line %d:" % (number + 1))
                print("  on disk: %s" % old)
                print("  source:  %s" % new)
                break
        print("run: python3 tools/make-code-map.py")
        return 1

    target.write_text(rendered, encoding="utf-8")
    print("wrote %s (%d lines)" % (target.relative_to(root), rendered.count("\n") + 1))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())