Hadrian Burkhardt
4706e2669c
An Android duplicate-photo finder. Pure-Kotlin core (SHA-256 + dHash hashing, exact and perceptual union-find grouping, three sensitivity presets) with MediaStore access, a persistent SQLite hash index and a tiered ScanEngine behind a Jetpack Compose UI. No INTERNET permission, no ads, no analytics, no account. Ships versionCode 2 / versionName 1.0.1, carrying the scan-stall hardening (BoundedCall breaker) and verified against a 5,363-photo device library. 80 JVM unit tests, lint clean. Release paperwork is in docs/ (Play checklist, store listing, edge-case testing), the upload-ready Play graphics and screenshots in store-assets/, and the generators for those assets in tools/.
104 lines
3.9 KiB
Python
104 lines
3.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Prepare an image folder for a *local* run of the app's grouping logic.
|
|
|
|
The duplicate engine (`core/`) is pure Kotlin, so it can be exercised on a desktop instead of a
|
|
phone. The only Android-specific parts are MediaStore, the image decoder and the hash index — and
|
|
the decoder's one subtlety is EXIF orientation.
|
|
|
|
So this script normalises orientation with Pillow (`ImageOps.exif_transpose`, the reference
|
|
behaviour) and writes lossless PNGs plus a manifest. `LocalGroupingTest` then fingerprints those
|
|
pixels with the app's own `PerceptualHash`, rebuilds the exact-duplicate path from the recorded
|
|
SHA-256, and runs the real `DuplicateGrouper`.
|
|
|
|
python3 tools/normalise-for-local-grouping.py fixtures/foto-dup-edge build/fotonorm
|
|
./gradlew :app:testDebugUnitTest --tests '*LocalGroupingTest*' -DfotoLocal.dir=/tmp/fotonorm
|
|
|
|
What this reproduces faithfully: the algorithm — hashing, the adaptive threshold, banded candidate
|
|
generation, the conjunction, union-find clustering, and the resulting groups.
|
|
What it does NOT reproduce: MediaStore's idea of what is indexed, the provider's thumbnail
|
|
decode, or the SQLite cache.
|
|
"""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from PIL import Image, ImageOps
|
|
|
|
SUPPORTED = {".jpg", ".jpeg", ".png", ".bmp", ".heic", ".heif", ".webp"}
|
|
|
|
|
|
def main() -> None:
|
|
if len(sys.argv) < 3:
|
|
print(__doc__)
|
|
raise SystemExit(2)
|
|
source = Path(os.path.expanduser(sys.argv[1]))
|
|
target = Path(os.path.expanduser(sys.argv[2]))
|
|
|
|
try:
|
|
import pillow_heif
|
|
|
|
pillow_heif.register_heif_opener()
|
|
except ImportError:
|
|
print("note: pillow-heif missing, HEIC files will be treated as unreadable")
|
|
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
for stale in target.glob("*.png"):
|
|
stale.unlink()
|
|
|
|
entries = []
|
|
for index, path in enumerate(sorted(source.iterdir())):
|
|
if not path.is_file() or path.suffix.lower() not in SUPPORTED:
|
|
continue
|
|
raw = path.read_bytes()
|
|
record = {
|
|
"name": path.name,
|
|
"sizeBytes": len(raw),
|
|
"modifiedSeconds": int(path.stat().st_mtime),
|
|
"sha256": hashlib.sha256(raw).hexdigest(),
|
|
}
|
|
try:
|
|
with Image.open(path) as img:
|
|
normalised = ImageOps.exif_transpose(img).convert("RGB")
|
|
out = target / f"{index:03d}.png"
|
|
normalised.save(out, "PNG")
|
|
record["normalised"] = out.name
|
|
record["width"], record["height"] = normalised.size
|
|
except Exception as error: # noqa: BLE001 - mirrors the app's "could not be read"
|
|
record["error"] = f"{type(error).__name__}: {error}"
|
|
entries.append(record)
|
|
|
|
(target / "manifest.json").write_text(json.dumps(entries, indent=1))
|
|
|
|
# Also a line-oriented manifest so the Kotlin side needs no JSON parser. One record per line:
|
|
# name, sizeBytes, modifiedSeconds, sha256, normalisedFile, width, height, error
|
|
tsv = target / "manifest.tsv"
|
|
with tsv.open("w") as handle:
|
|
for entry in entries:
|
|
handle.write(
|
|
"\t".join(
|
|
[
|
|
entry["name"],
|
|
str(entry["sizeBytes"]),
|
|
str(entry["modifiedSeconds"]),
|
|
entry["sha256"],
|
|
entry.get("normalised", ""),
|
|
str(entry.get("width", 0)),
|
|
str(entry.get("height", 0)),
|
|
entry.get("error", "").replace("\t", " ").replace("\n", " "),
|
|
]
|
|
)
|
|
+ "\n"
|
|
)
|
|
|
|
readable = sum(1 for e in entries if "error" not in e)
|
|
print(f"prepared {readable} readable of {len(entries)} files in {target}")
|
|
for entry in entries:
|
|
if "error" in entry:
|
|
print(f" unreadable: {entry['name']} ({entry['error'][:60]})")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|