#!/usr/bin/env python3 """Prepare an image folder for a *local* run of the app's grouping logic. The duplicate engine (`core/`) is pure Kotlin, so it can be exercised on a desktop instead of a phone. The only Android-specific parts are MediaStore, the image decoder and the hash index — and the decoder's one subtlety is EXIF orientation. So this script normalises orientation with Pillow (`ImageOps.exif_transpose`, the reference behaviour) and writes lossless PNGs plus a manifest. `LocalGroupingTest` then fingerprints those pixels with the app's own `PerceptualHash`, rebuilds the exact-duplicate path from the recorded SHA-256, and runs the real `DuplicateGrouper`. python3 tools/normalise-for-local-grouping.py fixtures/foto-dup-edge build/fotonorm ./gradlew :app:testDebugUnitTest --tests '*LocalGroupingTest*' -DfotoLocal.dir=/tmp/fotonorm What this reproduces faithfully: the algorithm — hashing, the adaptive threshold, banded candidate generation, the conjunction, union-find clustering, and the resulting groups. What it does NOT reproduce: MediaStore's idea of what is indexed, the provider's thumbnail decode, or the SQLite cache. """ import hashlib import json import os import sys from pathlib import Path from PIL import Image, ImageOps SUPPORTED = {".jpg", ".jpeg", ".png", ".bmp", ".heic", ".heif", ".webp"} def main() -> None: if len(sys.argv) < 3: print(__doc__) raise SystemExit(2) source = Path(os.path.expanduser(sys.argv[1])) target = Path(os.path.expanduser(sys.argv[2])) try: import pillow_heif pillow_heif.register_heif_opener() except ImportError: print("note: pillow-heif missing, HEIC files will be treated as unreadable") target.mkdir(parents=True, exist_ok=True) for stale in target.glob("*.png"): stale.unlink() entries = [] for index, path in enumerate(sorted(source.iterdir())): if not path.is_file() or path.suffix.lower() not in SUPPORTED: continue raw = path.read_bytes() record = { "name": path.name, "sizeBytes": len(raw), "modifiedSeconds": int(path.stat().st_mtime), "sha256": hashlib.sha256(raw).hexdigest(), } try: with Image.open(path) as img: normalised = ImageOps.exif_transpose(img).convert("RGB") out = target / f"{index:03d}.png" normalised.save(out, "PNG") record["normalised"] = out.name record["width"], record["height"] = normalised.size except Exception as error: # noqa: BLE001 - mirrors the app's "could not be read" record["error"] = f"{type(error).__name__}: {error}" entries.append(record) (target / "manifest.json").write_text(json.dumps(entries, indent=1)) # Also a line-oriented manifest so the Kotlin side needs no JSON parser. One record per line: # name, sizeBytes, modifiedSeconds, sha256, normalisedFile, width, height, error tsv = target / "manifest.tsv" with tsv.open("w") as handle: for entry in entries: handle.write( "\t".join( [ entry["name"], str(entry["sizeBytes"]), str(entry["modifiedSeconds"]), entry["sha256"], entry.get("normalised", ""), str(entry.get("width", 0)), str(entry.get("height", 0)), entry.get("error", "").replace("\t", " ").replace("\n", " "), ] ) + "\n" ) readable = sum(1 for e in entries if "error" not in e) print(f"prepared {readable} readable of {len(entries)} files in {target}") for entry in entries: if "error" in entry: print(f" unreadable: {entry['name']} ({entry['error'][:60]})") if __name__ == "__main__": main()