Initial commit: Foto Dup Finder 1.0.1

An Android duplicate-photo finder. Pure-Kotlin core (SHA-256 + dHash hashing, exact and
perceptual union-find grouping, three sensitivity presets) with MediaStore access, a
persistent SQLite hash index and a tiered ScanEngine behind a Jetpack Compose UI.
No INTERNET permission, no ads, no analytics, no account.

Ships versionCode 2 / versionName 1.0.1, carrying the scan-stall hardening (BoundedCall
breaker) and verified against a 5,363-photo device library. 80 JVM unit tests, lint clean.

Release paperwork is in docs/ (Play checklist, store listing, edge-case testing), the
upload-ready Play graphics and screenshots in store-assets/, and the generators for those
assets in tools/.
This commit is contained in:
Hadrian Burkhardt
2026-10-13 23:13:39 +02:00
commit 4706e2669c
100 changed files with 8998 additions and 0 deletions
+187
View File
@@ -0,0 +1,187 @@
#!/usr/bin/env python3
"""Generate hostile / edge-case image fixtures for Foto Dup Finder.
Every file here is something a real photo library can contain: an interrupted download, a
half-written JPEG, a crafted "decompression bomb", a photo whose pixels are fine but whose
EXIF says it is rotated, two byte-identical copies, and so on.
They are written to fixtures/foto-dup-edge/ inside the repository. That directory is
git-ignored — the fixtures are regenerated on demand, so no binary test data is committed:
python3 tools/make-edge-case-fixtures.py
adb push fixtures/foto-dup-edge /sdcard/DCIM/foto-dup-edge
Then scan in the app and check: nothing crashes, unreadable files are reported rather than
silently dropped, the EXIF variants of the same picture are *not* reported as duplicates of
each other (they normalise to the same pixels), and dup-a/dup-b *are*.
"""
import random
import struct
import zlib
from pathlib import Path
from PIL import Image
REPO = Path(__file__).resolve().parent.parent
OUT = REPO / "fixtures" / "foto-dup-edge"
random.seed(7)
def write(name: str, data: bytes) -> Path:
path = OUT / name
path.write_bytes(data)
return path
def gradient(width: int = 320, height: int = 240) -> Image.Image:
"""A structured base image — distinct enough from the solids to matter."""
img = Image.new("RGB", (width, height))
px = img.load()
for y in range(height):
for x in range(width):
px[x, y] = ((60 + x) % 256, (90 + y) % 256, 200)
return img
def png_chunk(kind: bytes, data: bytes) -> bytes:
return (
struct.pack(">I", len(data))
+ kind
+ data
+ struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
)
def heic_section(base: Image.Image) -> None:
"""HEIC samples, encoded locally so they carry no third-party licence.
Android can decode HEIC from API 28, so these exercise that path — and because we encode
them ourselves from our own images, they can be shipped/pushed freely.
"""
try:
import pillow_heif
pillow_heif.register_heif_opener()
except ImportError:
print("\nHEIC fixtures skipped — needs an encoder. Generate them with:")
print(" python3 -m venv /tmp/heicvenv")
print(" /tmp/heicvenv/bin/pip install pillow-heif")
print(" /tmp/heicvenv/bin/python tools/make-edge-case-fixtures.py")
return
# The iPhone default format: control, plus a byte-identical copy for the exact-duplicate path.
basic = OUT / "heic-basic.heic"
base.save(basic, "HEIF", quality=90)
write("heic-dup.heic", basic.read_bytes())
# Bigger than a thumbnail, so the subsampled fallback decode gets exercised.
gradient(2400, 1800).save(OUT / "heic-large.heic", "HEIF", quality=85)
# NOTE: HEIC EXIF *orientation* cannot be set with this encoder. Both `exif=<bytes>` and
# `exif=<Image.Exif>` are written, but the orientation tag comes back as 1 regardless, which
# would make a fixture that lies about what it tests. That one case therefore has to be
# checked with a real camera/iPhone HEIC — see docs/EDGE_CASE_TESTING.md.
# One scene encoded both ways: is a codec conversion still worth reporting?
scene = gradient(800, 600)
scene.save(OUT / "scene.jpg", "JPEG", quality=92)
scene.save(OUT / "scene.heic", "HEIF", quality=90)
# Near-identical solids in HEIC too — the band-index stress case.
for index in (0, 1):
Image.new("RGB", (400, 300), (20 + index, 40, 60)).save(
OUT / f"heic-solid-{index}.heic",
"HEIF",
quality=90,
)
def main() -> None:
OUT.mkdir(parents=True, exist_ok=True)
base = gradient()
# A distinct control scene, so the duplicate pair below is unambiguous. (Previously this was
# byte-identical to dup-a/dup-b, which produced a confusing group of three.)
control = gradient(240, 180)
control = control.transpose(Image.FLIP_LEFT_RIGHT)
write("valid.jpg", _jpeg_bytes(control))
good = _jpeg_bytes(base)
# 2. Interrupted download / half-written file: valid header, no end-of-image marker.
write("truncated.jpg", good[: len(good) * 4 // 10])
# 3. Right header, garbage body.
write("garbage-body.jpg", good[:120] + bytes(random.getrandbits(8) for _ in range(4096)))
# 4. Zero bytes. MediaStore normally filters these, but a sideloaded file can slip through.
write("zero-bytes.jpg", b"")
# 5. A PNG wearing a .jpg extension — the MIME type lies.
png_bytes = _png_bytes(base)
write("renamed-png.jpg", png_bytes)
# 6. A valid JPEG with no extension at all.
write("no-extension", good)
# 7. Decompression bomb: the header declares 60000 x 60000 (14 GB decoded) but carries
# almost no pixel data. A naive decoder allocates until it dies.
ihdr = struct.pack(">IIBBBBB", 60000, 60000, 8, 2, 0, 0, 0)
write(
"decompression-bomb.png",
b"\x89PNG\r\n\x1a\n"
+ png_chunk(b"IHDR", ihdr)
+ png_chunk(b"IDAT", zlib.compress(b"\x00" * 64))
+ png_chunk(b"IEND", b""),
)
# 8. The same picture written with every EXIF orientation value. All eight are the same
# photo and must normalise to the same fingerprint.
for orientation in range(1, 9):
exif = Image.Exif()
exif[0x0112] = orientation
path = OUT / f"exif-orientation-{orientation}.jpg"
base.save(path, "JPEG", quality=90, exif=exif)
# 9. Byte-identical copies — the exact-duplicate path must find these.
write("dup-a.jpg", good)
write("dup-b.jpg", good)
# 10. Odd but legal encodings that must not break the hasher.
base.convert("L").save(OUT / "grayscale.jpg", "JPEG", quality=90)
base.convert("RGBA").save(OUT / "with-alpha.png", "PNG")
Image.new("RGB", (1, 1), (255, 0, 0)).save(OUT / "tiny-1px.jpg", "JPEG")
# 11. Near-identical solids: colour-only similarity bait, and a stress case for the banded
# index (all of these may share hash bands).
for index in range(6):
Image.new("RGB", (400, 300), (20 + index, 40, 60)).save(
OUT / f"solid-{index}.png", "PNG"
)
total = sorted(OUT.iterdir())
heic_section(base)
total = sorted(OUT.iterdir())
print(f"wrote {len(total)} fixtures to {OUT}")
for path in total:
print(f" {path.stat().st_size:>9,} {path.name}")
def _jpeg_bytes(img: Image.Image) -> bytes:
import io
buffer = io.BytesIO()
img.save(buffer, "JPEG", quality=92)
return buffer.getvalue()
def _png_bytes(img: Image.Image) -> bytes:
import io
buffer = io.BytesIO()
img.save(buffer, "PNG")
return buffer.getvalue()
if __name__ == "__main__":
main()
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env bash
#
# Generates the Google Play store graphics from the app's own icon vectors.
#
# ./tools/make-store-assets.sh [output-dir] # default: ./store-assets
#
# Produces, matching Play's asset specs (support.google.com/.../answer/9866151):
# icon-512.png 512 x 512 32-bit PNG *with* alpha (ours is full-bleed,
# so every pixel ends up opaque anyway)
# feature-graphic-1024x500.png 1024 x 500 24-bit PNG, no alpha
#
# Output lands in ./store-assets/ *inside* the repo, never in $HOME: the editable sources are
# the SVGs in tools/store-assets/ and these PNGs are the files that get uploaded to the Play
# Console, so they belong with the release they were made for. Pass a directory to override.
set -euo pipefail
here="$(cd "$(dirname "$0")" && pwd)"
repo="$(dirname "$here")"
out="${1:-$repo/store-assets}"
mkdir -p "$out"
render() { # <svg> <width> <height> <filename> <keep-alpha|flatten>
local svg="$1" width="$2" height="$3" name="$4" mode="$5"
inkscape "$svg" \
--export-type=png --export-width="$width" --export-height="$height" \
--export-filename="$out/$name" >/dev/null 2>&1
if [ "$mode" = keep-alpha ]; then
# Play asks for a 32-bit PNG here. The SVG's background rect covers the whole
# canvas, so the alpha channel is fully opaque — no accidental transparency.
magick "$out/$name" -alpha on PNG32:"$out/$name"
else
magick "$out/$name" -background '#1B5E20' -alpha remove -alpha off PNG24:"$out/$name"
fi
printf ' %s ' "$out/$name"
identify -format '%wx%h %[channels] %b\n' "$out/$name"
}
echo "Writing store assets to $out"
render "$here/store-assets/icon.svg" 512 512 icon-512.png keep-alpha
render "$here/store-assets/feature-graphic.svg" 1024 500 feature-graphic-1024x500.png flatten
echo "Done. Screenshots: ./tools/make-store-screenshots.sh # reads store-assets/raw-screenshots/"
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env bash
#
# Turns raw device screenshots into Play-legal store screenshots.
#
# ./tools/make-store-screenshots.sh [raw-dir] [output-dir]
#
# Both paths default to inside the repo, never to $HOME:
# raw store-assets/raw-screenshots/ git-ignored staging for the `adb` captures
# output store-assets/screenshots/ the cropped files that get uploaded to Play
#
# Why this exists: Play rejects a screenshot whose longer side is more than twice its
# shorter side ("your screenshot's maximum dimension can't be more than twice its minimum
# dimension"). A 1080 x 2400 phone capture is 2.22:1, so it has to become 1080 x 2160.
#
# The crop removes system chrome only — the status bar at the top, the navigation bar at the
# bottom — never app content. Both heights were measured on the test device:
#
# STATUS_BAR_PX ITYPE_STATUS_BAR frame height 93 px (1080 x 2400 @ 440 dpi)
# NAV_BAR_PX ITYPE_NAVIGATION_BAR bottom inset 130 px (same device)
#
# adb shell dumpsys window displays | grep -oE 'ITYPE_STATUS_BAR frame=\[[^]]*\]\[[^]]*\]'
# adb shell dumpsys window displays | grep -oE 'control type=ITYPE_NAVIGATION_BAR.*mInsetsHint=\{[^}]*\}'
#
# Output is 24-bit PNG without alpha, which is what Play asks for. Captures are not shot by
# this script — take them yourself (the shot list is in docs/STORE_LISTING.md) and drop them
# into store-assets/raw-screenshots/, or pass the directory as the first argument.
set -euo pipefail
here="$(cd "$(dirname "$0")" && pwd)"
repo="$(dirname "$here")"
raw="${1:-$repo/store-assets/raw-screenshots}"
out="${2:-$repo/store-assets/screenshots}"
status_bar="${STATUS_BAR_PX:-93}"
nav_bar="${NAV_BAR_PX:-130}"
if [ ! -d "$raw" ]; then
echo "usage: $0 [raw-screenshot-dir] [output-dir]" >&2
echo "error: no such directory: $raw" >&2
echo " capture first, e.g. adb exec-out screencap -p > $raw/01-empty.png" >&2
exit 2
fi
mkdir -p "$out"
shopt -s nullglob
for f in "$raw"/*.png; do
name="$(basename "$f")"
width="$(identify -format '%w' "$f")"
height="$(identify -format '%h' "$f")"
# Drop the two system bars, then clamp to a 2:1 portrait frame.
keep=$(( height - status_bar - nav_bar ))
cap=$(( width * 2 ))
if [ "$keep" -gt "$cap" ]; then
keep="$cap"
fi
magick "$f" -crop "${width}x${keep}+0+${status_bar}" +repage \
-background '#FFFBFE' -alpha remove -alpha off PNG24:"$out/$name"
printf ' %s %s -> ' "$name" "$(identify -format '%wx%h' "$f")"
identify -format '%wx%h %[channels] %b\n' "$out/$name"
done
+103
View File
@@ -0,0 +1,103 @@
#!/usr/bin/env python3
"""Prepare an image folder for a *local* run of the app's grouping logic.
The duplicate engine (`core/`) is pure Kotlin, so it can be exercised on a desktop instead of a
phone. The only Android-specific parts are MediaStore, the image decoder and the hash index — and
the decoder's one subtlety is EXIF orientation.
So this script normalises orientation with Pillow (`ImageOps.exif_transpose`, the reference
behaviour) and writes lossless PNGs plus a manifest. `LocalGroupingTest` then fingerprints those
pixels with the app's own `PerceptualHash`, rebuilds the exact-duplicate path from the recorded
SHA-256, and runs the real `DuplicateGrouper`.
python3 tools/normalise-for-local-grouping.py fixtures/foto-dup-edge build/fotonorm
./gradlew :app:testDebugUnitTest --tests '*LocalGroupingTest*' -DfotoLocal.dir=/tmp/fotonorm
What this reproduces faithfully: the algorithm — hashing, the adaptive threshold, banded candidate
generation, the conjunction, union-find clustering, and the resulting groups.
What it does NOT reproduce: MediaStore's idea of what is indexed, the provider's thumbnail
decode, or the SQLite cache.
"""
import hashlib
import json
import os
import sys
from pathlib import Path
from PIL import Image, ImageOps
SUPPORTED = {".jpg", ".jpeg", ".png", ".bmp", ".heic", ".heif", ".webp"}
def main() -> None:
if len(sys.argv) < 3:
print(__doc__)
raise SystemExit(2)
source = Path(os.path.expanduser(sys.argv[1]))
target = Path(os.path.expanduser(sys.argv[2]))
try:
import pillow_heif
pillow_heif.register_heif_opener()
except ImportError:
print("note: pillow-heif missing, HEIC files will be treated as unreadable")
target.mkdir(parents=True, exist_ok=True)
for stale in target.glob("*.png"):
stale.unlink()
entries = []
for index, path in enumerate(sorted(source.iterdir())):
if not path.is_file() or path.suffix.lower() not in SUPPORTED:
continue
raw = path.read_bytes()
record = {
"name": path.name,
"sizeBytes": len(raw),
"modifiedSeconds": int(path.stat().st_mtime),
"sha256": hashlib.sha256(raw).hexdigest(),
}
try:
with Image.open(path) as img:
normalised = ImageOps.exif_transpose(img).convert("RGB")
out = target / f"{index:03d}.png"
normalised.save(out, "PNG")
record["normalised"] = out.name
record["width"], record["height"] = normalised.size
except Exception as error: # noqa: BLE001 - mirrors the app's "could not be read"
record["error"] = f"{type(error).__name__}: {error}"
entries.append(record)
(target / "manifest.json").write_text(json.dumps(entries, indent=1))
# Also a line-oriented manifest so the Kotlin side needs no JSON parser. One record per line:
# name, sizeBytes, modifiedSeconds, sha256, normalisedFile, width, height, error
tsv = target / "manifest.tsv"
with tsv.open("w") as handle:
for entry in entries:
handle.write(
"\t".join(
[
entry["name"],
str(entry["sizeBytes"]),
str(entry["modifiedSeconds"]),
entry["sha256"],
entry.get("normalised", ""),
str(entry.get("width", 0)),
str(entry.get("height", 0)),
entry.get("error", "").replace("\t", " ").replace("\n", " "),
]
)
+ "\n"
)
readable = sum(1 for e in entries if "error" not in e)
print(f"prepared {readable} readable of {len(entries)} files in {target}")
for entry in entries:
if "error" in entry:
print(f" unreadable: {entry['name']} ({entry['error'][:60]})")
if __name__ == "__main__":
main()
+40
View File
@@ -0,0 +1,40 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1024" height="500" viewBox="0 0 1024 500">
<!--
Play feature graphic (1024 x 500, no transparency).
Same palette as the app icon (#1B5E20) with the icon motif on the left and the
store copy on the right. Text stays inside the middle band because Play crops the
graphic to a square for some placements.
-->
<defs>
<linearGradient id="bg" x1="0" y1="0" x2="1" y2="1">
<stop offset="0" stop-color="#0D3D13" />
<stop offset="0.55" stop-color="#1B5E20" />
<stop offset="1" stop-color="#2E7D32" />
</linearGradient>
</defs>
<rect x="0" y="0" width="1024" height="500" fill="url(#bg)" />
<!-- Icon motif: the two photo frames, centred on (210, 250) at ~300 px wide. -->
<g transform="translate(210,250) scale(7.46) translate(-49.8,-46.85)">
<g transform="translate(30,22) scale(1.9)">
<path d="M4,5 h16 a1,1 0 0 1 1,1 v12 a1,1 0 0 1 -1,1 h-16 a1,1 0 0 1 -1,-1 v-12 a1,1 0 0 1 1,-1 z"
fill="#FFFFFF" fill-opacity="0.45" />
</g>
<g transform="translate(24,28) scale(1.9)">
<path d="M4,5 h16 a1,1 0 0 1 1,1 v12 a1,1 0 0 1 -1,1 h-16 a1,1 0 0 1 -1,-1 v-12 a1,1 0 0 1 1,-1 z"
fill="none" stroke="#FFFFFF" stroke-width="1.6" />
<path d="M7.5,9 m-1.4,0 a1.4,1.4 0 1 0 2.8,0 a1.4,1.4 0 1 0 -2.8,0" fill="#FFFFFF" />
<path d="M4,17 l4,-5 l3,3.5 l3.5,-4.5 l5.5,6 z" fill="#FFFFFF" />
</g>
</g>
<!-- Store copy -->
<text x="420" y="205" font-family="DejaVu Sans" font-weight="bold" font-size="56"
fill="#FFFFFF">Foto Dup Finder</text>
<text x="422" y="258" font-family="DejaVu Sans" font-size="25"
fill="#FFFFFF" fill-opacity="0.92">Find and remove duplicate photos</text>
<text x="422" y="296" font-family="DejaVu Sans" font-size="25"
fill="#FFFFFF" fill-opacity="0.92">Fast, ad-free and completely offline</text>
</svg>

After

Width:  |  Height:  |  Size: 1.9 KiB

+24
View File
@@ -0,0 +1,24 @@
<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 108 108">
<!--
The Play high-res icon: the app's adaptive icon, full bleed.
Mirrors app/src/main/res/drawable/ic_launcher_{background,foreground}.xml — the
foreground's two groups are copied verbatim, including their scale/translate, so the
icon a store listing shows matches the launcher icon exactly.
-->
<rect x="0" y="0" width="108" height="108" fill="#1B5E20" />
<!-- Back frame (semi-transparent) -->
<g transform="translate(30,22) scale(1.9)">
<path d="M4,5 h16 a1,1 0 0 1 1,1 v12 a1,1 0 0 1 -1,1 h-16 a1,1 0 0 1 -1,-1 v-12 a1,1 0 0 1 1,-1 z"
fill="#FFFFFF" fill-opacity="0.45" />
</g>
<!-- Front frame outline, sun and mountain -->
<g transform="translate(24,28) scale(1.9)">
<path d="M4,5 h16 a1,1 0 0 1 1,1 v12 a1,1 0 0 1 -1,1 h-16 a1,1 0 0 1 -1,-1 v-12 a1,1 0 0 1 1,-1 z"
fill="none" stroke="#FFFFFF" stroke-width="1.6" />
<path d="M7.5,9 m-1.4,0 a1.4,1.4 0 1 0 2.8,0 a1.4,1.4 0 1 0 -2.8,0" fill="#FFFFFF" />
<path d="M4,17 l4,-5 l3,3.5 l3.5,-4.5 l5.5,6 z" fill="#FFFFFF" />
</g>
</svg>

After

Width:  |  Height:  |  Size: 1.1 KiB