imgduptective 0.1.0__tar.gz

imgduptective-0.1.0/PKG-INFO

@@ -0,0 +1,17 @@
+Metadata-Version: 2.1
+Name: imgduptective
+Version: 0.1.0
+Summary: Image Duplicates Detective
+Author-Email: sacha <sachahony@gmail.com>, Sacha Hony <zazahohonini@gmail.com>
+License: MIT
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Project-URL: Homepage, https://github.com/zazaho/imgduptective
+Requires-Python: >=3.10
+Requires-Dist: Pillow
+Description-Content-Type: text/markdown
+
+# Image Duplicate Detectiver (imgduptective)
+
+# Description

imgduptective-0.1.0/README.md

@@ -0,0 +1,3 @@
+# Image Duplicate Detectiver (imgduptective)
+
+# Description

imgduptective-0.1.0/imgduptective/imgduptective.py

@@ -0,0 +1,530 @@
+#!/usr/bin/env python3
+"""imgduptective - non-interactive duplicate image detector using gradient horizontal hash."""
+import argparse
+import hashlib
+import sqlite3
+import sys
+from collections import Counter
+from multiprocessing import Pool
+from pathlib import Path
+
+from PIL import Image
+
+# --- Image handling ---
+
+_EXIF_ORIENTATION_CODE = 274
+_EXIF_ORIENTATION_CORRECTION_MAPPING = {
+    2: Image.FLIP_LEFT_RIGHT,
+    3: Image.ROTATE_180,
+    4: Image.FLIP_TOP_BOTTOM,
+    5: Image.TRANSPOSE,
+    6: Image.ROTATE_270,
+    7: Image.TRANSVERSE,
+    8: Image.ROTATE_90,
+}
+_TABLE16 = [i / 256 for i in range(65536)]
+
+
+def image_open(fn):
+    try:
+        img = Image.open(fn)
+        if img.format == "PNG" and img.mode == "I" and max(img.getdata()) > 255:
+            img = img.point(_TABLE16, "L")
+    except Exception:
+        return None
+    try:
+        exif = img._getexif()
+        if exif and _EXIF_ORIENTATION_CODE in exif:
+            code = exif[_EXIF_ORIENTATION_CODE]
+            if code in _EXIF_ORIENTATION_CORRECTION_MAPPING:
+                img = img.transpose(_EXIF_ORIENTATION_CORRECTION_MAPPING[code])
+    except Exception:
+        pass
+    return img
+
+
+# --- Hash functions ---
+
+def dhash_horizontal(img):
+    try:
+        i9x8 = img.convert("L").resize((9, 8), Image.BOX)
+    except:
+        return None
+    h = 0
+    for y in range(8):
+        for x in range(8):
+            if i9x8.getpixel((x + 1, y)) > i9x8.getpixel((x, y)):
+                h |= 1 << (y * 8 + x)
+    return h
+
+# --- Worker functions for multiprocessing ---
+
+def _compute_filehash(filepath_str):
+    hasher = hashlib.sha1()
+    with open(filepath_str, "rb") as f:
+        while chunk := f.read(65536):
+            hasher.update(chunk)
+    return (filepath_str, hasher.hexdigest())
+
+
+def _compute_imagehash(filepath_str):
+    img = image_open(filepath_str)
+    if img is None:
+        return (filepath_str, None)
+    return (filepath_str, dhash_horizontal(img))
+
+
+def _compare_range(args):
+    """Compare a range of the triangular matrix."""
+    row_start, row_end, hashes, threshold = args
+    matches = []
+    for i in range(row_start, row_end):
+        h_i = hashes[i]
+        for j in range(i + 1, len(hashes)):
+            if (h_i ^ hashes[j]).bit_count() <= threshold:
+                matches.append((i, j))
+    return matches
+
+
+# --- Database ---
+
+DB_PATH = Path.home() / ".config" / "imgduptective" / "imgduptective.db"
+
+
+def get_db():
+    DB_PATH.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(DB_PATH)
+    conn.execute(
+        "CREATE TABLE IF NOT EXISTS HashValueTable "
+        "(id integer PRIMARY KEY, FileHash text NOT NULL UNIQUE, "
+        "ImageHashValue text NOT NULL)"
+    )
+    conn.execute(
+        "CREATE TABLE IF NOT EXISTS FileTable "
+        "(id integer PRIMARY KEY, FilePath text NOT NULL UNIQUE, "
+        "FileHash text NOT NULL, ImageHashValue text)"
+    )
+    conn.commit()
+    return conn
+
+
+def db_get_imagehashes(conn, filehashes):
+    """Batch lookup: return dict of filehash -> imagehash (int) for those found."""
+    result = {}
+    filehashes = list(filehashes)
+    batch_size = 500
+    for i in range(0, len(filehashes), batch_size):
+        batch = filehashes[i:i + batch_size]
+        placeholders = ",".join("?" * len(batch))
+        rows = conn.execute(
+            f"SELECT FileHash, ImageHashValue FROM HashValueTable WHERE FileHash IN ({placeholders})",
+            batch,
+        ).fetchall()
+        for filehash, imagehash_hex in rows:
+            result[filehash] = int(imagehash_hex, 16)
+    return result
+
+
+def db_set_imagehashes(conn, pairs):
+    """Batch insert: pairs is list of (filehash, imagehash_int)."""
+    conn.executemany(
+        "INSERT OR IGNORE INTO HashValueTable (FileHash, ImageHashValue) VALUES(?, ?)",
+        [(fh, format(ih, "016x")) for fh, ih in pairs],
+    )
+    conn.commit()
+
+
+def db_upsert_files(conn, rows):
+    """Batch upsert: rows is list of (filepath, filehash, imagehash_hex_or_none)."""
+    conn.executemany(
+        "INSERT INTO FileTable (FilePath, FileHash, ImageHashValue) VALUES(?, ?, ?) "
+        "ON CONFLICT(FilePath) DO UPDATE SET FileHash=excluded.FileHash, ImageHashValue=excluded.ImageHashValue",
+        rows,
+    )
+    conn.commit()
+
+
+def db_prune_missing(conn):
+    """Remove FileTable entries where file no longer exists. Process in batches."""
+    cursor = conn.execute("SELECT id, FilePath FROM FileTable")
+    missing = []
+    while True:
+        rows = cursor.fetchmany(1000)
+        if not rows:
+            break
+        missing.extend(r[0] for r in rows if not Path(r[1]).exists())
+    if missing:
+        conn.executemany("DELETE FROM FileTable WHERE id=?", [(i,) for i in missing])
+    conn.commit()
+
+
+# --- Main logic ---
+
+PHOTO_EXTENSIONS = {".jpg", ".jpeg", ".png", ".heic", ".heif", ".webp", ".tiff", ".tif", ".bmp", ".gif"}
+
+
+def find_files(root, photos_only=False):
+    """Walk root, yield regular files (no symlinks)."""
+    for path in Path(root).rglob("*"):
+        if path.is_file() and not path.is_symlink():
+            if photos_only and path.suffix.lower() not in PHOTO_EXTENSIONS:
+                continue
+            yield path
+
+
+def progress(current, total, label="Progress"):
+    pct = current * 100 // total
+    print(f"\r{label}: {pct}%", end="", file=sys.stderr, flush=True)
+    if current == total:
+        print(file=sys.stderr)
+
+
+def load_entries(files, conn):
+    """Compute filehashes in parallel, lookup/compute imagehashes, return entries dict."""
+    file_strs = [str(f) for f in files]
+    total = len(file_strs)
+    print(f"Found {total} files", file=sys.stderr)
+
+    # Step 1: compute file hashes in parallel (streaming)
+    print("\rHashing files...", end="", file=sys.stderr, flush=True)
+    with Pool() as pool:
+        filehash_results = pool.map(_compute_filehash, file_strs)
+    print("\rHashing files... done", file=sys.stderr)
+
+    # filepath_str -> filehash
+    filehash_map = dict(filehash_results)
+
+    # Step 2: batch DB lookup for existing imagehashes
+    unique_filehashes = set(filehash_map.values())
+    known = db_get_imagehashes(conn, unique_filehashes)
+
+    # Step 3: compute missing imagehashes in parallel
+    need_compute = [fp for fp, fh in filehash_map.items() if fh not in known]
+    print(f"{len(need_compute)} files need image hash computation", file=sys.stderr)
+    if need_compute:
+        print(f"\rComputing image hashes ({len(need_compute)} files)...", end="", file=sys.stderr, flush=True)
+        with Pool() as pool:
+            imagehash_results = pool.map(_compute_imagehash, need_compute)
+        print(f"\rComputing image hashes ({len(need_compute)} files)... done", file=sys.stderr)
+
+        # Store new hashes in DB (batch)
+        new_pairs = []
+        for filepath_str, imagehash in imagehash_results:
+            if imagehash is not None:
+                filehash = filehash_map[filepath_str]
+                known[filehash] = imagehash
+                new_pairs.append((filehash, imagehash))
+        if new_pairs:
+            db_set_imagehashes(conn, new_pairs)
+
+    # Step 4: build entries and batch upsert to FileTable
+    entries = {}
+    upsert_rows = []
+    for filepath_str, filehash in filehash_map.items():
+        imagehash = known.get(filehash)
+        imagehash_hex = format(imagehash, "016x") if imagehash is not None else None
+        upsert_rows.append((filepath_str, filehash, imagehash_hex))
+        if imagehash is not None:
+            entries[filepath_str] = (filehash, imagehash)
+    db_upsert_files(conn, upsert_rows)
+
+    return entries
+
+
+def find_duplicates(entries, threshold):
+    """Find near-duplicate groups using multiprocessing for comparison."""
+    filepaths = list(entries.keys())
+    n = len(filepaths)
+    # Flat list of hashes indexed by position
+    hashes = [entries[fp][1] for fp in filepaths]
+
+    if n < 2:
+        return []
+
+    total_pairs = n * (n - 1) // 2
+    print(f"\rComparing {total_pairs} pairs...", end="", file=sys.stderr, flush=True)
+
+    # Split rows across workers (each worker handles a range of 'i' values)
+    num_workers = min(8, n)
+    chunk_size = max(1, n // num_workers)
+    ranges = []
+    for start in range(0, n, chunk_size):
+        end = min(start + chunk_size, n)
+        ranges.append((start, end, hashes, threshold))
+
+    with Pool() as pool:
+        results = pool.map(_compare_range, ranges)
+    print(f"\rComparing {total_pairs} pairs... done", file=sys.stderr)
+
+    # Merge results into groups using filepath strings
+    groups = {}
+    for matches in results:
+        for i, j in matches:
+            a, b = filepaths[i], filepaths[j]
+            groups.setdefault(a, {a}).add(b)
+            groups.setdefault(b, {b}).add(a)
+
+    # Remove subgroups
+    unique_groups = []
+    for g in sorted(groups.values(), key=len, reverse=True):
+        if not any(g <= existing for existing in unique_groups):
+            unique_groups.append(g)
+
+    return unique_groups
+
+
+def view_groups(groups):
+    """Simple tkinter viewer to browse duplicate groups side by side."""
+    import gzip
+    import shutil
+    import tkinter as tk
+
+    groups = [sorted(g) for g in groups]
+    cwd = str(Path.cwd())
+    state = {"idx": 0}
+
+    def rel_path(filepath):
+        """Show path relative to cwd, or enough to distinguish."""
+        if filepath.startswith(cwd):
+            return filepath[len(cwd):].lstrip("/")
+        return filepath
+
+    def gzip_delete(filepath):
+        """Compress file with gzip and remove the original."""
+        with open(filepath, "rb") as f_in:
+            with gzip.open(filepath + ".gz", "wb") as f_out:
+                shutil.copyfileobj(f_in, f_out)
+        Path(filepath).unlink()
+
+    root = tk.Tk()
+    root.title("imgduptective viewer")
+    root.geometry("1200x700")
+
+    top_frame = tk.Frame(root)
+    top_frame.pack(fill="x")
+    label = tk.Label(top_frame, text="", font=("", 12))
+    label.pack(side="left", padx=10)
+    help_label = tk.Label(top_frame, text="←/→:navigate  d:delete(gzip)  q:quit", font=("", 9))
+    help_label.pack(side="right", padx=10)
+
+    canvas_frame = tk.Frame(root)
+    canvas_frame.pack(fill="both", expand=True)
+
+    def show_group():
+        for w in canvas_frame.winfo_children():
+            w.destroy()
+
+        idx = state["idx"]
+        group = groups[idx]
+        label.config(text=f"Group {idx + 1}/{len(groups)} ({len(group)} files)")
+
+        root.update_idletasks()
+        n = len(group)
+        max_w = max(100, (root.winfo_width() - 20) // n - 10)
+        max_h = root.winfo_height() - 100
+
+        state["photos"] = []
+        state["frames"] = []
+        state["selected"] = set()
+
+        for i, filepath in enumerate(group):
+            frame = tk.Frame(canvas_frame, borderwidth=2, relief="groove")
+            frame.pack(side="left", padx=5, pady=5, fill="both", expand=True)
+            state["frames"].append(frame)
+
+            img = image_open(filepath)
+            if img:
+                ratio = min(max_w / img.size[0], max_h / img.size[1])
+                if ratio < 1:
+                    img = img.resize((int(img.size[0] * ratio), int(img.size[1] * ratio)), Image.LANCZOS)
+                from PIL import ImageTk
+                photo = ImageTk.PhotoImage(img)
+                state["photos"].append(photo)
+                img_label = tk.Label(frame, image=photo)
+                img_label.pack()
+                img_label.bind("<Button-1>", lambda e, idx=i: toggle_select(idx))
+
+            tk.Label(frame, text=rel_path(filepath), wraplength=max_w, font=("", 9)).pack()
+
+    def toggle_select(i):
+        if i in state["selected"]:
+            state["selected"].discard(i)
+            state["frames"][i].config(relief="groove", bg="#d9d9d9")
+        else:
+            state["selected"].add(i)
+            state["frames"][i].config(relief="solid", bg="#ffcccc")
+
+    def delete_selected(event=None):
+        idx = state["idx"]
+        group = groups[idx]
+        to_delete = sorted(state["selected"], reverse=True)
+        if not to_delete:
+            return
+        for i in to_delete:
+            filepath = group[i]
+            try:
+                gzip_delete(filepath)
+            except Exception:
+                pass
+        # Remove deleted from group
+        groups[idx] = [f for i, f in enumerate(group) if i not in state["selected"]]
+        # Remove empty groups
+        if len(groups[idx]) < 2:
+            groups.pop(idx)
+            if not groups:
+                root.destroy()
+                return
+            state["idx"] = state["idx"] % len(groups)
+        show_group()
+
+    def next_group(event=None):
+        state["idx"] = (state["idx"] + 1) % len(groups)
+        show_group()
+
+    def prev_group(event=None):
+        state["idx"] = (state["idx"] - 1) % len(groups)
+        show_group()
+
+    root.bind("<Right>", next_group)
+    root.bind("<n>", next_group)
+    root.bind("<space>", next_group)
+    root.bind("<Left>", prev_group)
+    root.bind("<p>", prev_group)
+    root.bind("<d>", delete_selected)
+    root.bind("<Delete>", delete_selected)
+    root.bind("<q>", lambda e: root.destroy())
+    root.bind("<Escape>", lambda e: root.destroy())
+
+    show_group()
+    root.mainloop()
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Detect near-duplicate images.")
+    parser.add_argument("threshold", type=int, nargs="?", help="Maximum hamming distance to consider a match")
+    parser.add_argument("--view", action="store_true", help="Open the builtin viewer to inspect duplicate groups")
+    parser.add_argument("--stats", action="store_true", help="Show per-directory duplicate statistics")
+    parser.add_argument("--check", action="store_true", help="Check what duplicates would be added without modifying the database")
+    parser.add_argument("--add", action="store_true", help="Add hashes to the database without comparing")
+    parser.add_argument("--photos", action="store_true", help="Only process common photo file formats")
+    parser.add_argument("--exact", action="store_true", help="Show only exact file matches instead of similar")
+    args = parser.parse_args()
+
+    if not args.add and not args.exact and args.threshold is None:
+        parser.error("threshold is required unless --add or --exact is used")
+
+    threshold = args.threshold
+    conn = get_db()
+
+    if args.check:
+        existing = {}
+        for row in conn.execute("SELECT FilePath, ImageHashValue FROM FileTable WHERE ImageHashValue IS NOT NULL").fetchall():
+            existing[row[0]] = int(row[1], 16)
+
+        files = list(find_files(Path.cwd(), args.photos))
+        file_strs = [str(f) for f in files]
+
+        with Pool() as pool:
+            filehash_results = pool.map(_compute_filehash, file_strs)
+
+        filehash_map = dict(filehash_results)
+        known = db_get_imagehashes(conn, set(filehash_map.values()))
+
+        need_compute = [fp for fp, fh in filehash_map.items() if fh not in known]
+        if need_compute:
+            with Pool() as pool:
+                imagehash_results = pool.map(_compute_imagehash, need_compute)
+            for fp, ih in imagehash_results:
+                if ih is not None:
+                    known[filehash_map[fp]] = ih
+
+        new_files = {}
+        for fp, fh in filehash_map.items():
+            ih = known.get(fh)
+            if ih is not None:
+                new_files[fp] = ih
+
+        found = False
+        for new_path, new_hash in new_files.items():
+            matches = [ex_path for ex_path, ex_hash in existing.items()
+                       if ex_path != new_path and bin(new_hash ^ ex_hash).count("1") <= threshold]
+            if matches:
+                if not found:
+                    print("=== New duplicates if added ===")
+                    found = True
+                print(f"{new_path}\t->\t" + "\t".join(sorted(matches)))
+
+        conn.close()
+        return
+
+    if args.add:
+        files = list(find_files(Path.cwd(), args.photos))
+        load_entries(files, conn)
+        db_prune_missing(conn)
+        conn.close()
+        return
+
+    files = list(find_files(Path.cwd(), args.photos))
+    entries = load_entries(files, conn)
+    db_prune_missing(conn)
+
+    # Report exact duplicates (same filehash)
+    hash_to_files = {}
+    for filepath, (filehash, imagehash) in entries.items():
+        hash_to_files.setdefault(filehash, []).append(filepath)
+    exact_dupes = [sorted(v) for v in hash_to_files.values() if len(v) > 1]
+
+    # Find near-duplicate groups
+    unique_groups = find_duplicates(entries, threshold) if not args.exact else []
+
+    # --stats: only show statistics
+    if args.stats:
+        total = len(entries)
+        dir_total = Counter()
+        for filepath in entries:
+            dir_total[str(Path(filepath).parent)] += 1
+
+        if args.exact:
+            dupes = sum(len(g) for g in exact_dupes)
+            print(f"Exact duplicates: {dupes}/{total}")
+            dir_dupes = Counter()
+            for group in exact_dupes:
+                for filepath in group:
+                    dir_dupes[str(Path(filepath).parent)] += 1
+        else:
+            all_dupes = {fp for group in unique_groups for fp in group}
+            print(f"Near-duplicates: {len(all_dupes)}/{total}")
+            dir_dupes = Counter()
+            for filepath in all_dupes:
+                dir_dupes[str(Path(filepath).parent)] += 1
+        if dir_dupes:
+            for dirname, count in dir_dupes.most_common():
+                print(f"{count}/{dir_total[dirname]}\t{dirname}")
+        conn.close()
+        return
+
+    # --view: open builtin viewer
+    groups_to_view = exact_dupes if args.exact else unique_groups
+    if args.view:
+        if groups_to_view:
+            view_groups(groups_to_view)
+        conn.close()
+        return
+
+    # Output groups
+    if args.exact:
+        if exact_dupes:
+            print("=== Exact duplicates ===")
+            for files in sorted(exact_dupes, key=lambda x: x[0]):
+                print("\t".join(files))
+    else:
+        if unique_groups:
+            print("=== Near-duplicate groups ===")
+            for group in unique_groups:
+                print("\t".join(sorted(group)))
+
+    conn.close()
+
+
+if __name__ == "__main__":
+    main()

imgduptective-0.1.0/pyproject.toml

@@ -0,0 +1,35 @@
+[project]
+name = "imgduptective"
+version = "0.1.0"
+description = "Image Duplicates Detective"
+authors = [
+    { name = "sacha", email = "sachahony@gmail.com" },
+    { name = "Sacha Hony", email = "zazahohonini@gmail.com" },
+]
+dependencies = [
+    "Pillow",
+]
+requires-python = ">=3.10"
+readme = "README.md"
+classifiers = [
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+]
+
+[project.license]
+text = "MIT"
+
+[project.urls]
+Homepage = "https://github.com/zazaho/imgduptective"
+
+[project.scripts]
+imgduptective = "imgduptective.imgduptective:main"
+
+[build-system]
+requires = [
+    "pdm-backend",
+]
+build-backend = "pdm.backend"
+
+[tool]