image-bridge-toolkit 0.1.0__py3-none-any.whl

image_bridge_toolkit/__init__.py

@@ -0,0 +1,5 @@
+"""image_bridge_toolkit: perceptual image fingerprint cache + cross-dataset bridge matcher."""
+from ._imaging_backend import HAS_OPENCV
+
+__version__ = "0.1.0"
+__all__ = ["HAS_OPENCV", "__version__"]

image_bridge_toolkit/_imaging_backend.py

@@ -0,0 +1,12 @@
+"""
+Single source of truth for the optional OpenCV backend.
+
+Every other module imports HAS_OPENCV / cv2 from here instead of doing its
+own try/except, so the fallback decision is made exactly once per process.
+"""
+try:
+    import cv2  # noqa: F401
+    HAS_OPENCV = True
+except ImportError:  # pragma: no cover - exercised only when cv2 absent
+    cv2 = None
+    HAS_OPENCV = False

image_bridge_toolkit/cache_builder.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""
+imgcache-build: production entry point for (re)generating image_cache.json
+files across a directory tree.
+
+Pipeline:
+  1. scanner.scan_tree      -> fast, single-pass directory discovery + mtimes
+  2. cache_io.load_cache    -> per-directory cache load (corruption-safe)
+  3. planner.plan_entry     -> per-file: which calc groups actually need work
+  4. worker processes       -> run only the needed calculators, in parallel
+  5. cache_io.write_cache_atomic -> durable, corruption-proof write back
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import time
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from pathlib import Path
+from typing import Any, Dict, FrozenSet, List, Tuple
+
+from PIL import Image
+
+from . import calculators
+from .cache_io import load_cache, write_cache_atomic
+from .constants import ALL_GROUPS
+from .logging_utils import get_logger
+from .planner import plan_entry
+from .scanner import scan_tree
+
+log = get_logger(__name__)
+
+# What a worker process needs to do one file, and what it hands back.
+_Job = Tuple[str, FrozenSet[str]]  # (image_path, groups_to_compute)
+
+
+def _compute_groups(image_path: str, groups: FrozenSet[str]) -> Dict[str, Any]:
+    """Runs in a worker process. Never raises -- an unreadable/corrupt image
+    yields an empty update rather than killing the whole batch."""
+    if not groups:
+        return {}
+    try:
+        with Image.open(image_path) as img:
+            img.load()
+            out: Dict[str, Any] = {}
+            if "hash" in groups:
+                out.update(calculators.compute_hashes(img))
+            if "sig" in groups:
+                out.update(calculators.compute_visual_signature(img))
+            if "lab" in groups:
+                out.update(calculators.compute_lab_metrics(img))
+            if "waveform" in groups:
+                out.update(calculators.compute_hist_waveform(img))
+            return out
+    except Exception as exc:
+        log.error("Failed to process %s: %s", image_path, exc)
+        return {}
+
+
+def _worker(job: _Job) -> Tuple[str, Dict[str, Any]]:
+    path, groups = job
+    return path, _compute_groups(path, groups)
+
+
+class ImageCacheBuilder:
+    def __init__(self, root_dir: str, workers: int = 0, force: bool = False):
+        self.root_dir = Path(root_dir).resolve()
+        self.workers = workers or max(1, os.cpu_count() or 1)
+        self.force = force
+
+    def run(self) -> Dict[str, int]:
+        stats = {"dirs": 0, "files_seen": 0, "files_updated": 0, "files_reused": 0}
+        t0 = time.time()
+
+        for dir_path, images in scan_tree(self.root_dir):
+            stats["dirs"] += 1
+            cache = {} if self.force else load_cache(dir_path)
+            new_cache: Dict[str, Any] = {}
+            jobs: List[_Job] = []
+            job_meta: Dict[str, Dict[str, Any]] = {}
+
+            for img_path, mtime in images:
+                stats["files_seen"] += 1
+                fname = img_path.name
+                needed, base_entry = plan_entry(None if self.force else cache.get(fname), mtime)
+                if not needed:
+                    new_cache[fname] = base_entry
+                    stats["files_reused"] += 1
+                    continue
+                jobs.append((str(img_path), needed))
+                job_meta[str(img_path)] = {"fname": fname, "base": base_entry}
+
+            if jobs:
+                new_cache.update(self._run_jobs(jobs, job_meta))
+                stats["files_updated"] += len(jobs)
+
+            # Only touch disk if something actually changed for this dir.
+            if jobs or len(new_cache) != len(cache) or self.force:
+                write_cache_atomic(dir_path, new_cache)
+
+            log.info("%s: %d image(s), %d updated", dir_path, len(images), len(jobs))
+
+        log.info(
+            "Done in %.1fs - dirs=%d files_seen=%d updated=%d reused=%d",
+            time.time() - t0, stats["dirs"], stats["files_seen"],
+            stats["files_updated"], stats["files_reused"],
+        )
+        return stats
+
+    def _run_jobs(self, jobs: List[_Job], job_meta: Dict[str, Dict[str, Any]]) -> Dict[str, Any]:
+        results: Dict[str, Any] = {}
+        if self.workers <= 1 or len(jobs) < 4:
+            for path, groups in jobs:
+                meta = job_meta[path]
+                entry = dict(meta["base"])
+                entry.update(_compute_groups(path, groups))
+                results[meta["fname"]] = entry
+            return results
+
+        with ProcessPoolExecutor(max_workers=self.workers) as pool:
+            futures = {pool.submit(_worker, job): job[0] for job in jobs}
+            for fut in as_completed(futures):
+                path = futures[fut]
+                meta = job_meta[path]
+                try:
+                    _, computed = fut.result()
+                except Exception as exc:
+                    log.error("Worker failed for %s: %s", path, exc)
+                    computed = {}
+                entry = dict(meta["base"])
+                entry.update(computed)
+                results[meta["fname"]] = entry
+        return results
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="imgcache-build", description="Build/update image_cache.json across a directory tree.")
+    p.add_argument("root", help="Root directory to scan (recursively).")
+    p.add_argument("-w", "--workers", type=int, default=0, help="Process pool size (default: CPU count).")
+    p.add_argument("--force", action="store_true", help="Ignore existing caches and recompute everything.")
+    p.add_argument("-v", "--verbose", action="store_true")
+    return p
+
+
+def main(argv: List[str] = None) -> int:
+    args = build_parser().parse_args(argv)
+    if args.verbose:
+        get_logger(__name__, verbose=True)
+    root = Path(args.root)
+    if not root.is_dir():
+        print(f"Not a directory: {root}", file=sys.stderr)
+        return 2
+    builder = ImageCacheBuilder(str(root), workers=args.workers, force=args.force)
+    builder.run()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

image_bridge_toolkit/cache_io.py

@@ -0,0 +1,49 @@
+"""Loading / writing image_cache.json with corruption tolerance and atomic writes."""
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from pathlib import Path
+from typing import Any, Dict
+
+from .constants import CACHE_FILE_NAME
+from .logging_utils import get_logger
+
+log = get_logger(__name__)
+
+
+def load_cache(dir_path: Path) -> Dict[str, Any]:
+    """Returns {} for a missing OR corrupt cache file. Corruption is logged,
+    never raised -- a bad JSON file must not abort the whole run, and per the
+    spec it is treated exactly like "no cache" (everything gets recomputed)."""
+    cache_path = dir_path / CACHE_FILE_NAME
+    if not cache_path.exists():
+        return {}
+    try:
+        with open(cache_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        if not isinstance(data, dict):
+            raise ValueError("cache root is not a JSON object")
+        return data
+    except Exception as exc:
+        log.warning("Corrupt cache at %s (%s) - treating as empty", cache_path, exc)
+        return {}
+
+
+def write_cache_atomic(dir_path: Path, cache: Dict[str, Any]) -> None:
+    dir_path.mkdir(parents=True, exist_ok=True)
+    cache_path = dir_path / CACHE_FILE_NAME
+    fd, tmp_name = tempfile.mkstemp(prefix=f".{CACHE_FILE_NAME}.", suffix=".tmp", dir=str(dir_path))
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            json.dump(cache, f, indent=2)
+            f.flush()
+            os.fsync(f.fileno())
+        os.replace(tmp_name, cache_path)
+    except Exception:
+        try:
+            os.unlink(tmp_name)
+        except OSError:
+            pass
+        raise

image_bridge_toolkit/calculators.py

@@ -0,0 +1,153 @@
+"""
+Perceptual fingerprint calculators. Algorithms are kept numerically
+equivalent to the original gridview.py reference implementation so that
+existing caches / downstream consumers stay compatible.
+
+Every function is pure (path/PIL.Image in -> plain-python-serialisable
+values out) and never raises: on internal failure it returns a neutral
+zero-value so a single unreadable image can't take down a batch job.
+"""
+from __future__ import annotations
+
+import math
+from collections import defaultdict
+from typing import Any, Dict, List, Tuple
+
+import numpy as np
+from PIL import Image
+
+from ._imaging_backend import HAS_OPENCV, cv2
+
+HASH_SIZE = 8
+PHASH_IMG_SIZE = 32
+SIG_POINTS = 16
+LAB_RESIZE = 64
+WAVEFORM_BINS = 256
+WAVEFORM_KEEP = 16
+
+
+def _dct_phash_pil(image: Image.Image, hash_size: int = HASH_SIZE) -> str:
+    try:
+        img = image.convert("L").resize((PHASH_IMG_SIZE, PHASH_IMG_SIZE), Image.Resampling.LANCZOS)
+        pixels = np.asarray(img, dtype=np.float64)
+        dct_abs = np.abs(np.fft.fft2(pixels))
+        dct_slice = dct_abs[1:hash_size + 1, 1:hash_size + 1].flatten()
+        median_val = np.median(dct_slice)
+        bits = (dct_slice > median_val).astype(np.uint8)
+        hex_str = "".join(f"{int(''.join(map(str, bits[i:i + 4])), 2):x}" for i in range(0, 64, 4))
+        return hex_str.zfill(16)
+    except Exception:
+        return "0" * 16
+
+
+def compute_hashes(image: Image.Image) -> Dict[str, str]:
+    """phash_luma / phash_color from an already-open PIL image."""
+    luma = _dct_phash_pil(image)
+    color = luma
+    try:
+        hsv = image.convert("HSV")
+        hue_channel = Image.fromarray(np.asarray(hsv)[:, :, 0])
+        color = _dct_phash_pil(hue_channel)
+    except Exception:
+        pass
+    return {"phash_luma": luma, "phash_color": color}
+
+
+def compute_visual_signature(image: Image.Image, num_points: int = SIG_POINTS) -> Dict[str, Any]:
+    try:
+        small = image.resize((16, 16), resample=Image.Resampling.LANCZOS)
+        pixels = np.array(small.convert("RGB")).reshape(-1, 3) / 255.0
+        coords = np.mgrid[0:1:16j, 0:1:16j].reshape(2, -1).T
+
+        selected = [int(np.argmax(np.std(pixels, axis=1)))]
+        for _ in range(num_points - 1):
+            c_dists = np.min([np.linalg.norm(pixels - pixels[s], axis=1) for s in selected], axis=0)
+            s_dists = np.min([np.linalg.norm(coords - coords[s], axis=1) for s in selected], axis=0)
+            selected.append(int(np.argmax(c_dists * s_dists)))
+
+        weights: Dict[int, int] = defaultdict(int)
+        for p in pixels:
+            dists = [np.linalg.norm(p - pixels[s]) for s in selected]
+            weights[int(np.argmin(dists))] += 1
+
+        sig = []
+        for i, idx in enumerate(selected):
+            r, g, b = (pixels[idx] * 255).astype(int)
+            sig.append({
+                "hex": f"#{r:02x}{g:02x}{b:02x}",
+                "x": round(float(coords[idx][1]) * 100, 1),
+                "y": round(float(coords[idx][0]) * 100, 1),
+                "size": round((weights[i] / 256) * 100, 1),
+            })
+        return {"visual_sig": sig}
+    except Exception:
+        return {"visual_sig": []}
+
+
+def _srgb_to_linear(c: np.ndarray) -> np.ndarray:
+    a = 0.055
+    return np.where(c <= 0.04045, c / 12.92, ((c + a) / (1 + a)) ** 2.4)
+
+
+def _rgb_to_lab(arr: np.ndarray) -> np.ndarray:
+    lin = _srgb_to_linear(arr)
+    m = np.array([[0.4124564, 0.3575761, 0.1804375],
+                  [0.2126729, 0.7151522, 0.0721750],
+                  [0.0193339, 0.1191920, 0.9503041]])
+    xyz = (lin.reshape(-1, 3) @ m.T).reshape(lin.shape)
+    xn, yn, zn = 0.95047, 1.0, 1.08883
+    xyz[..., 0] /= xn
+    xyz[..., 1] /= yn
+    xyz[..., 2] /= zn
+    t = np.where(xyz > 0.008856, np.cbrt(xyz), 7.787 * xyz + 16 / 116)
+    L = 116 * t[..., 1] - 16
+    a = 500 * (t[..., 0] - t[..., 1])
+    b = 200 * (t[..., 1] - t[..., 2])
+    return np.stack([L, a, b], axis=-1)
+
+
+def compute_lab_metrics(image: Image.Image) -> Dict[str, float]:
+    try:
+        img = image.convert("RGB").resize((LAB_RESIZE, LAB_RESIZE), Image.Resampling.LANCZOS)
+        arr = np.array(img, dtype=np.float64) / 255.0
+        lab = _rgb_to_lab(arr)
+        L, a, b = tuple(lab.mean(axis=(0, 1)))
+        return {
+            "lab_value": round(float(L), 2),
+            "lab_hue": round(float(np.arctan2(b, a)), 2),
+            "lab_chroma": round(float(math.sqrt(a ** 2 + b ** 2)), 2),
+            "lab_warmth": round(float(b), 2),
+        }
+    except Exception:
+        return {"lab_value": 0.0, "lab_hue": 0.0, "lab_chroma": 0.0, "lab_warmth": 0.0}
+
+
+def _dct_1d_numpy(hist_norm: np.ndarray) -> np.ndarray:
+    n = len(hist_norm)
+    out = np.zeros(n, dtype=np.float64)
+    k = np.arange(n)
+    factor = np.pi / (2.0 * n)
+    # vectorised type-II DCT (equivalent to the reference's nested-loop version)
+    n_idx = np.arange(n).reshape(-1, 1)
+    cos_table = np.cos((2 * n_idx + 1) * k.reshape(1, -1) * factor)
+    s = hist_norm @ cos_table
+    c = np.full(n, math.sqrt(2.0 / n))
+    c[0] = math.sqrt(1.0 / n)
+    return c * s
+
+
+def compute_hist_waveform(image: Image.Image) -> Dict[str, List[float]]:
+    try:
+        gray = np.array(image.convert("L"), dtype=np.uint8)
+        hist, _ = np.histogram(gray, bins=WAVEFORM_BINS, range=(0, WAVEFORM_BINS))
+        total = hist.sum()
+        hist_norm = (hist.astype(np.float32) / total) if total > 0 else np.zeros(WAVEFORM_BINS, dtype=np.float32)
+
+        if HAS_OPENCV:
+            dct_hist = cv2.dct(hist_norm.reshape(-1, 1)).flatten()
+        else:
+            dct_hist = _dct_1d_numpy(hist_norm.astype(np.float64))
+
+        return {"hist_waveform": [round(float(v), 6) for v in dct_hist[:WAVEFORM_KEEP]]}
+    except Exception:
+        return {"hist_waveform": [0.0] * WAVEFORM_KEEP}

image_bridge_toolkit/constants.py

@@ -0,0 +1,17 @@
+CACHE_FILE_NAME = "image_cache.json"
+IGNORE_MARKER = ".ignore_subdir"
+VALID_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".tiff"}
+
+# Field groups. A cache entry is only "complete" once every group's keys are
+# present; mtime mismatch invalidates ALL groups (the file content changed),
+# while a missing group with a matching mtime just means the schema grew
+# (e.g. you added hist_waveform after caches already existed) and only that
+# group needs to be (re)computed.
+GROUP_KEYS = {
+    "hash": ("phash_luma", "phash_color"),
+    "sig": ("visual_sig",),
+    "lab": ("lab_value", "lab_hue", "lab_chroma", "lab_warmth"),
+    "waveform": ("hist_waveform",),
+}
+ALL_GROUPS = frozenset(GROUP_KEYS.keys())
+REQUIRED_KEYS = frozenset(k for keys in GROUP_KEYS.values() for k in keys)

image_bridge_toolkit/dataset_loader.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""Loads all image_cache.json files under a root into flat numpy vector arrays
+for the matcher, skipping .ignore_subdir trees exactly like the builder does."""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any, Dict, List
+
+import numpy as np
+
+from .cache_io import load_cache
+from .constants import CACHE_FILE_NAME, IGNORE_MARKER
+from .logging_utils import get_logger
+
+log = get_logger(__name__)
+
+
+class Dataset:
+    __slots__ = ("paths", "phash_luma", "phash_color", "waveform", "lab", "signatures")
+
+    def __init__(self, paths, phash_luma, phash_color, waveform, lab, signatures):
+        self.paths = paths
+        self.phash_luma = phash_luma
+        self.phash_color = phash_color
+        self.waveform = waveform
+        self.lab = lab
+        self.signatures = signatures
+
+    def __len__(self) -> int:
+        return len(self.paths)
+
+
+def load_dataset(root_dir: Path) -> Dataset:
+    paths: List[str] = []
+    phashes_luma: List[int] = []
+    phashes_color: List[int] = []
+    waveforms: List[List[float]] = []
+    labs: List[List[float]] = []
+    signatures: List[List[Dict[str, Any]]] = []
+
+    root_dir = Path(root_dir).resolve()
+    stack = [root_dir]
+    while stack:
+        current = stack.pop()
+        try:
+            with os.scandir(current) as it:
+                entries = list(it)
+        except OSError as exc:
+            log.warning("Cannot read directory %s (%s) - skipping", current, exc)
+            continue
+
+        if any(e.name == IGNORE_MARKER and e.is_file(follow_symlinks=False) for e in entries):
+            continue
+
+        has_cache = any(e.name == CACHE_FILE_NAME for e in entries)
+        for e in entries:
+            if e.is_dir(follow_symlinks=False):
+                stack.append(Path(e.path))
+
+        if not has_cache:
+            continue
+
+        cache = load_cache(current)
+        for fname, meta in cache.items():
+            try:
+                record = (
+                    str(current / fname),
+                    int(meta.get("phash_luma", "0") or "0", 16),
+                    int(meta.get("phash_color", meta.get("phash_luma", "0")) or "0", 16),
+                    meta.get("hist_waveform", [0.0] * 16) or [0.0] * 16,
+                    [meta.get("lab_value", 0.0), meta.get("lab_hue", 0.0),
+                     meta.get("lab_chroma", 0.0), meta.get("lab_warmth", 0.0)],
+                    meta.get("visual_sig", []) or [],
+                )
+            except (TypeError, ValueError) as exc:
+                log.warning("Skipping malformed cache entry %s/%s: %s", current, fname, exc)
+                continue
+            path, ph_luma, ph_color, wf, lab, sig = record
+            paths.append(path)
+            phashes_luma.append(ph_luma)
+            phashes_color.append(ph_color)
+            waveforms.append(wf)
+            labs.append(lab)
+            signatures.append(sig)
+
+    return Dataset(
+        paths=paths,
+        phash_luma=np.array(phashes_luma, dtype=np.uint64),
+        phash_color=np.array(phashes_color, dtype=np.uint64),
+        waveform=np.array(waveforms, dtype=np.float32) if waveforms else np.zeros((0, 16), dtype=np.float32),
+        lab=np.array(labs, dtype=np.float32) if labs else np.zeros((0, 4), dtype=np.float32),
+        signatures=signatures,
+    )

image_bridge_toolkit/logging_utils.py

@@ -0,0 +1,15 @@
+import logging
+import sys
+
+
+def get_logger(name: str, verbose: bool = False) -> logging.Logger:
+    logger = logging.getLogger(name)
+    if not logger.handlers:
+        handler = logging.StreamHandler(sys.stderr)
+        handler.setFormatter(
+            logging.Formatter("%(asctime)s %(levelname)-7s %(name)s: %(message)s", "%H:%M:%S")
+        )
+        logger.addHandler(handler)
+    logger.setLevel(logging.DEBUG if verbose else logging.INFO)
+    logger.propagate = False
+    return logger

image_bridge_toolkit/matcher.py

@@ -0,0 +1,240 @@
+"""
+imgmatch-bridge: cache-backed multi-tier matcher that builds a bridge map
+from a (watermarked / derivative) dataset to a much larger originals dataset.
+
+Tiers, cheapest -> most expensive, each narrowing the candidate pool:
+  1. Dual pHash (luma+color) Hamming distance -- vectorised over the whole
+     originals array, this is what makes 1,000 x 50,000 tractable at all.
+  2. DCT histogram waveform distance (within Tier-1 survivors).
+  3. LAB perceptual colour distance.
+  4. Visual-signature spatial/colour layout distance (watermark-robust).
+  5. Optional direct pixel SSIM re-ranking of the surviving near-ties.
+
+Output is streamed to disk as JSON Lines (one query result per line) so a
+50k x 1k run is resumable and never holds the whole result set in memory,
+and a crash partway through doesn't lose completed work.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from pathlib import Path
+from typing import Any, Dict, List, Set
+
+import numpy as np
+
+from .dataset_loader import Dataset, load_dataset
+from .logging_utils import get_logger
+from .similarity import direct_ssim, hamming_matrix, visual_sig_distance
+
+log = get_logger(__name__)
+
+
+# Populated once per worker process via ProcessPoolExecutor(initializer=...)
+# so the (potentially large, 50k-entry) datasets are pickled to each worker
+# exactly once, not re-serialised on every one of the 1,000 submitted tasks.
+_WORKER_DA: Dataset = None
+_WORKER_ORIG: Dataset = None
+_WORKER_ARGS: Dict[str, Any] = {}
+
+
+def _init_worker(da: Dataset, orig: Dataset, max_hamming: int, candidate_threshold: float) -> None:
+    global _WORKER_DA, _WORKER_ORIG, _WORKER_ARGS
+    _WORKER_DA, _WORKER_ORIG = da, orig
+    _WORKER_ARGS = {"max_hamming": max_hamming, "candidate_threshold": candidate_threshold}
+
+
+def _worker_match(q_idx: int) -> Dict[str, Any]:
+    return match_one(q_idx, _WORKER_DA, _WORKER_ORIG, **_WORKER_ARGS)
+
+
+def _already_done(output_path: Path) -> Set[str]:
+    done: Set[str] = set()
+    if not output_path.exists():
+        return done
+    with open(output_path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                done.add(json.loads(line)["query_path"])
+            except Exception:
+                continue
+    return done
+
+
+def match_one(
+    q_idx: int,
+    da: Dataset,
+    orig: Dataset,
+    max_hamming: int,
+    candidate_threshold: float,
+) -> Dict[str, Any]:
+    q_path = da.paths[q_idx]
+    q_luma = int(da.phash_luma[q_idx])
+    q_color = int(da.phash_color[q_idx])
+    q_wf = da.waveform[q_idx]
+    q_lab = da.lab[q_idx]
+    q_sig = da.signatures[q_idx]
+
+    dist_luma = hamming_matrix(q_luma, orig.phash_luma)
+    dist_color = hamming_matrix(q_color, orig.phash_color)
+    hamming_combined = 0.7 * dist_luma + 0.3 * dist_color
+
+    cand_indices = np.where(dist_luma <= max_hamming)[0]
+    if len(cand_indices) == 0:
+        return {"query_path": q_path, "top_matches": []}
+
+    wf_dists = np.linalg.norm(orig.waveform[cand_indices] - q_wf, axis=1)
+    lab_dists = np.linalg.norm(orig.lab[cand_indices] - q_lab, axis=1) / 100.0
+
+    scores = []
+    for pos, orig_idx in enumerate(cand_indices):
+        h_dist = hamming_combined[orig_idx]
+        w_dist = wf_dists[pos]
+        l_dist = lab_dists[pos]
+
+        s_h = max(0.0, 1.0 - (h_dist / 20.0))
+        s_w = max(0.0, 1.0 - (w_dist / 0.5))
+        s_l = max(0.0, 1.0 - l_dist)
+
+        sig_dist = visual_sig_distance(q_sig, orig.signatures[orig_idx])
+        s_sig = max(0.0, 1.0 - sig_dist)
+
+        composite = float(0.35 * s_h + 0.35 * s_w + 0.15 * s_l + 0.15 * s_sig)
+        scores.append({
+            "orig_idx": int(orig_idx),
+            "match_path": orig.paths[orig_idx],
+            "confidence": composite,
+            "metrics": {
+                "hamming_luma": int(dist_luma[orig_idx]),
+                "hamming_color": int(dist_color[orig_idx]),
+                "waveform_dist": round(float(w_dist), 6),
+                "lab_dist": round(float(l_dist), 6),
+                "sig_dist": round(float(sig_dist), 6),
+            },
+        })
+
+    scores.sort(key=lambda c: c["confidence"], reverse=True)
+    best = scores[0]["confidence"]
+    eligible = [c for c in scores if (best - c["confidence"]) <= candidate_threshold]
+    return {"query_path": q_path, "top_matches": eligible, "_q_idx": q_idx}
+
+
+def _finalize(result: Dict[str, Any], da: Dataset, direct_compare: bool) -> Dict[str, Any]:
+    q_idx = result.pop("_q_idx", None)
+    if direct_compare and result["top_matches"] and q_idx is not None:
+        q_path = da.paths[q_idx]
+        for c in result["top_matches"]:
+            ssim_val = direct_ssim(q_path, c["match_path"])
+            c["metrics"]["ssim"] = round(ssim_val, 4)
+            c["confidence"] = round(0.5 * c["confidence"] + 0.5 * ssim_val, 4)
+        result["top_matches"].sort(key=lambda c: c["confidence"], reverse=True)
+    else:
+        for c in result["top_matches"]:
+            c["confidence"] = round(c["confidence"], 4)
+    for c in result["top_matches"]:
+        c.pop("orig_idx", None)
+    return result
+
+
+def run_match(
+    da_root: str,
+    orig_root: str,
+    output_path: str,
+    max_hamming: int = 16,
+    candidate_threshold: float = 0.03,
+    do_direct_compare: bool = False,
+    workers: int = 0,
+    resume: bool = True,
+) -> None:
+    da = load_dataset(Path(da_root))
+    orig = load_dataset(Path(orig_root))
+    log.info("Loaded %d derivative images, %d original images", len(da), len(orig))
+
+    if len(da) == 0 or len(orig) == 0:
+        log.warning("One of the datasets is empty (no image_cache.json entries found) - nothing to do.")
+        Path(output_path).touch()
+        return
+
+    out_path = Path(output_path)
+    done = _already_done(out_path) if resume else set()
+    if done:
+        log.info("Resuming: %d/%d already completed", len(done), len(da))
+
+    pending = [i for i in range(len(da)) if da.paths[i] not in done]
+    t0 = time.time()
+
+    with open(out_path, "a", encoding="utf-8") as out_f:
+        if workers and workers > 1 and not do_direct_compare:
+            # Direct-compare (SSIM) reads image bytes and is worth parallelising
+            # separately; the vectorised numpy stages below are already fast
+            # enough single-threaded and cheaper to run in-process.
+            with ProcessPoolExecutor(
+                max_workers=workers,
+                initializer=_init_worker,
+                initargs=(da, orig, max_hamming, candidate_threshold),
+            ) as pool:
+                futures = {pool.submit(_worker_match, i): i for i in pending}
+                for n, fut in enumerate(as_completed(futures), 1):
+                    result = _finalize(fut.result(), da, do_direct_compare)
+                    out_f.write(json.dumps(result) + "\n")
+                    out_f.flush()
+                    if n % 50 == 0:
+                        log.info("Matched %d/%d", n, len(pending))
+        else:
+            for n, i in enumerate(pending, 1):
+                result = match_one(i, da, orig, max_hamming, candidate_threshold)
+                result = _finalize(result, da, do_direct_compare)
+                out_f.write(json.dumps(result) + "\n")
+                out_f.flush()
+                if n % 50 == 0:
+                    log.info("Matched %d/%d", n, len(pending))
+
+    log.info("Done in %.1fs -> %s", time.time() - t0, out_path)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="imgmatch-bridge", description="Build a confidence-scored bridge map between two cached image datasets.")
+    p.add_argument("da_root", help="Root of the derivative/watermarked dataset (e.g. DeviantArt scrape).")
+    p.add_argument("orig_root", help="Root of the originals dataset.")
+    p.add_argument("-o", "--output", default="bridge_map.jsonl", help="Output JSONL path (default: bridge_map.jsonl).")
+    p.add_argument("--max-hamming", type=int, default=16, help="Tier-1 pHash-luma Hamming cutoff (default: 16).")
+    p.add_argument("--candidate-threshold", type=float, default=0.03, help="Keep candidates within this confidence gap of the best match (default: 0.03).")
+    p.add_argument("--direct-compare", action="store_true", help="Run Tier-5 pixel SSIM re-ranking on surviving candidates (slow).")
+    p.add_argument("-w", "--workers", type=int, default=0, help="Process pool size for per-query matching (default: single process).")
+    p.add_argument("--no-resume", action="store_true", help="Ignore/overwrite any existing output file instead of resuming.")
+    p.add_argument("-v", "--verbose", action="store_true")
+    return p
+
+
+def main(argv: List[str] = None) -> int:
+    args = build_parser().parse_args(argv)
+    if args.verbose:
+        get_logger(__name__, verbose=True)
+
+    if args.no_resume and Path(args.output).exists():
+        Path(args.output).unlink()
+
+    for root_name, root in (("da_root", args.da_root), ("orig_root", args.orig_root)):
+        if not Path(root).is_dir():
+            print(f"Not a directory: {root} ({root_name})", file=sys.stderr)
+            return 2
+
+    run_match(
+        args.da_root, args.orig_root, args.output,
+        max_hamming=args.max_hamming,
+        candidate_threshold=args.candidate_threshold,
+        do_direct_compare=args.direct_compare,
+        workers=args.workers,
+        resume=not args.no_resume,
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

image_bridge_toolkit/planner.py

@@ -0,0 +1,33 @@
+"""
+Decides, per image, which calculation groups (if any) need to run.
+
+Rule (this is the spec you gave, made explicit):
+  - No cache entry at all                -> compute every group.
+  - Cache entry exists but mtime differs -> the file content changed, so ALL
+    previously-computed groups are stale and must be recomputed. (A naive
+    "only fill in missing keys" approach would silently keep stale phash/lab/
+    etc. values for a changed file just because the keys happen to already
+    exist -- that's a correctness bug, not a caching optimisation, so it's
+    deliberately not what this does.)
+  - Cache entry exists and mtime matches -> only groups whose keys are
+    missing get (re)computed; everything else is reused untouched. This is
+    what lets you cheaply backfill a new field type across a huge existing
+    cache without re-hashing every image.
+"""
+from __future__ import annotations
+
+from typing import Any, Dict, FrozenSet, Optional, Tuple
+
+from .constants import ALL_GROUPS, GROUP_KEYS
+
+
+def plan_entry(cached_entry: Optional[Dict[str, Any]], mtime: int) -> Tuple[FrozenSet[str], Dict[str, Any]]:
+    """Returns (groups_needing_compute, base_entry_to_extend)."""
+    if not cached_entry or cached_entry.get("mtime") != mtime:
+        return ALL_GROUPS, {"mtime": mtime}
+
+    missing = frozenset(
+        group for group, keys in GROUP_KEYS.items()
+        if not all(k in cached_entry for k in keys)
+    )
+    return missing, dict(cached_entry)

image_bridge_toolkit/scanner.py

@@ -0,0 +1,58 @@
+"""
+Fast filesystem discovery.
+
+Why this is faster than a plain os.walk + Path.stat() pass: os.scandir's
+DirEntry caches the stat result from the readdir call on most platforms, so
+entry.stat() below is (usually) free -- no extra stat(2) syscall per file the
+way Path(f).stat() or os.stat(f) would cost. We grab mtime here, once, while
+we're already iterating the directory to decide what's an image / a
+subdirectory / the ignore marker, instead of doing a second full walk later
+just to stat files for cache invalidation.
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Iterator, List, Tuple
+
+from .constants import IGNORE_MARKER, VALID_EXTENSIONS
+from .logging_utils import get_logger
+
+log = get_logger(__name__)
+
+# (absolute path, mtime as int seconds)
+ImageEntry = Tuple[Path, int]
+
+
+def scan_tree(root_dir: Path) -> Iterator[Tuple[Path, List[ImageEntry]]]:
+    """Yields (directory, [ (image_path, mtime), ... ]) for every directory
+    under root_dir that contains at least one image, skipping any directory
+    (and everything beneath it) that contains an IGNORE_MARKER file."""
+    stack = [Path(root_dir)]
+    while stack:
+        current = stack.pop()
+        try:
+            with os.scandir(current) as it:
+                entries = list(it)
+        except OSError as exc:
+            log.warning("Cannot read directory %s (%s) - skipping", current, exc)
+            continue
+
+        if any(e.name == IGNORE_MARKER and e.is_file(follow_symlinks=False) for e in entries):
+            continue  # this directory and its subtree are opted out
+
+        images: List[ImageEntry] = []
+        for entry in entries:
+            try:
+                if entry.is_dir(follow_symlinks=False):
+                    stack.append(Path(entry.path))
+                    continue
+                suffix = os.path.splitext(entry.name)[1].lower()
+                if suffix in VALID_EXTENSIONS:
+                    mtime = int(entry.stat(follow_symlinks=False).st_mtime)
+                    images.append((Path(entry.path), mtime))
+            except OSError as exc:
+                log.warning("Cannot stat %s (%s) - skipping", entry.path, exc)
+
+        if images:
+            yield current, images

image_bridge_toolkit/similarity.py

@@ -0,0 +1,81 @@
+"""Distance / similarity primitives used by the multi-tier matcher."""
+from __future__ import annotations
+
+from typing import Any, Dict, List
+
+import numpy as np
+
+from ._imaging_backend import HAS_OPENCV, cv2
+
+if not HAS_OPENCV:
+    from PIL import Image
+
+
+def hamming_matrix(query_hash: int, dataset_hashes: np.ndarray) -> np.ndarray:
+    """Vectorised popcount of (query XOR every hash in dataset_hashes)."""
+    xor = np.bitwise_xor(dataset_hashes, np.uint64(query_hash))
+    dist = np.zeros(len(dataset_hashes), dtype=np.int32)
+    for b in range(64):
+        dist += ((xor >> np.uint64(b)) & np.uint64(1)).astype(np.int32)
+    return dist
+
+
+def _hex_to_rgb(hex_str: str) -> np.ndarray:
+    h = hex_str.lstrip("#")
+    return np.array([int(h[i:i + 2], 16) for i in (0, 2, 4)], dtype=np.float32)
+
+
+def visual_sig_distance(sig1: List[Dict[str, Any]], sig2: List[Dict[str, Any]]) -> float:
+    """Spatial+colour layout discrepancy; border points (watermark zones,
+    x/y outside 10-90%) are dropped so DeviantArt watermarks/frames don't
+    dominate the score."""
+    if not sig1 or not sig2:
+        return 1.0
+
+    p1 = [pt for pt in sig1 if 10.0 <= pt["x"] <= 90.0 and 10.0 <= pt["y"] <= 90.0] or sig1
+    p2 = [pt for pt in sig2 if 10.0 <= pt["x"] <= 90.0 and 10.0 <= pt["y"] <= 90.0] or sig2
+
+    rgb2 = np.array([_hex_to_rgb(pt["hex"]) for pt in p2])
+    xy2 = np.array([[pt["x"], pt["y"]] for pt in p2], dtype=np.float32)
+
+    total = 0.0
+    for pt1 in p1:
+        rgb1 = _hex_to_rgb(pt1["hex"])
+        xy1 = np.array([pt1["x"], pt1["y"]], dtype=np.float32)
+        c_dist = np.linalg.norm(rgb2 - rgb1, axis=1) / 441.67
+        s_dist = np.linalg.norm(xy2 - xy1, axis=1) / 141.42
+        combined = 0.6 * c_dist + 0.4 * s_dist
+        total += float(np.min(combined))
+    return total / len(p1)
+
+
+def direct_ssim(img_path1: str, img_path2: str) -> float:
+    """Optional, expensive pixel-level structural similarity for the final
+    verification tier. Never raises - returns 0.0 (no similarity) on any
+    read/decode failure so a bad file can't crash a batch run."""
+    try:
+        if HAS_OPENCV:
+            i1 = cv2.imread(img_path1, cv2.IMREAD_GRAYSCALE)
+            i2 = cv2.imread(img_path2, cv2.IMREAD_GRAYSCALE)
+            if i1 is None or i2 is None:
+                return 0.0
+            i2 = cv2.resize(i2, (i1.shape[1], i1.shape[0]), interpolation=cv2.INTER_AREA)
+            c1, c2 = (0.01 * 255) ** 2, (0.03 * 255) ** 2
+            i1 = i1.astype(np.float64)
+            i2 = i2.astype(np.float64)
+            mu1 = cv2.GaussianBlur(i1, (11, 11), 1.5)
+            mu2 = cv2.GaussianBlur(i2, (11, 11), 1.5)
+            mu1_sq, mu2_sq, mu1_mu2 = mu1 ** 2, mu2 ** 2, mu1 * mu2
+            sigma1_sq = cv2.GaussianBlur(i1 ** 2, (11, 11), 1.5) - mu1_sq
+            sigma2_sq = cv2.GaussianBlur(i2 ** 2, (11, 11), 1.5) - mu2_sq
+            sigma12 = cv2.GaussianBlur(i1 * i2, (11, 11), 1.5) - mu1_mu2
+            ssim_map = ((2 * mu1_mu2 + c1) * (2 * sigma12 + c2)) / ((mu1_sq + mu2_sq + c1) * (sigma1_sq + sigma2_sq + c2))
+            return float(np.mean(ssim_map))
+        else:
+            with Image.open(img_path1) as im1, Image.open(img_path2) as im2:
+                g1 = np.array(im1.convert("L").resize((256, 256)), dtype=np.float64)
+                g2 = np.array(im2.convert("L").resize((256, 256)), dtype=np.float64)
+                mse = np.mean((g1 - g2) ** 2)
+                return max(0.0, 1.0 - (mse / 65535.0))
+    except Exception:
+        return 0.0

image_bridge_toolkit-0.1.0.dist-info/METADATA

@@ -0,0 +1,234 @@
+Metadata-Version: 2.4
+Name: image-bridge-toolkit
+Version: 0.1.0
+Summary: Cache-backed perceptual image fingerprinting and cross-dataset bridge matching.
+Author: Andrew Kingdom
+License: MIT for Code / CC BY-NC-ND 4.0 for Documentation
+Project-URL: Homepage, https://github.com/akingdom/image_bridge_toolkit
+Project-URL: Repository, https://github.com/akingdom/image_bridge_toolkit
+Project-URL: Issues, https://github.com/akingdom/image_bridge_toolkit/issues
+Keywords: image-processing,perceptual-hash,phash,deduplication,dct,computer-vision,reverse-image-search
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Scientific/Engineering :: Image Processing
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: numpy>=1.23
+Requires-Dist: Pillow>=9.0
+Provides-Extra: opencv
+Requires-Dist: opencv-python-headless>=4.6; extra == "opencv"
+Provides-Extra: progress
+Requires-Dist: tqdm>=4.65; extra == "progress"
+Provides-Extra: all
+Requires-Dist: opencv-python-headless>=4.6; extra == "all"
+Requires-Dist: tqdm>=4.65; extra == "all"
+Dynamic: license-file
+
+<p align="center">
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 600" width="100%" height="400">
+    <defs>
+      <linearGradient id="leftNodeGrad" x1="0%" y1="0%" x2="100%" y2="100%">
+        <stop offset="0%" stop-color="#06B6D4" stop-opacity="0.25"/>
+        <stop offset="100%" stop-color="#0F172A" stop-opacity="0.8"/>
+      </linearGradient>
+
+      <linearGradient id="rightNodeGrad" x1="0%" y1="0%" x2="100%" y2="100%">
+        <stop offset="0%" stop-color="#EC4899" stop-opacity="0.3"/>
+        <stop offset="100%" stop-color="#06B6D4" stop-opacity="0.15"/>
+      </linearGradient>
+
+      <linearGradient id="bridgeGrad" x1="0%" y1="0%" x2="100%" y2="0%">
+        <stop offset="0%" stop-color="#06B6D4"/>
+        <stop offset="50%" stop-color="#FFFFFF"/>
+        <stop offset="100%" stop-color="#EC4899"/>
+      </linearGradient>
+
+      <filter id="glow" x="-20%" y="-20%" width="140%" height="140%">
+        <feGaussianBlur stdDeviation="8" result="blur"/>
+        <feMerge>
+          <feMergeNode in="blur"/>
+          <feMergeNode in="SourceGraphic"/>
+        </feMerge>
+      </filter>
+
+      <filter id="glow-subtle" x="-20%" y="-20%" width="140%" height="140%">
+        <feGaussianBlur stdDeviation="4" result="blur"/>
+        <feMerge>
+          <feMergeNode in="blur"/>
+          <feMergeNode in="SourceGraphic"/>
+        </feMerge>
+      </filter>
+
+      <pattern id="fingerprintGrid" width="16" height="16" patternUnits="userSpaceOnUse">
+        <path d="M 16 0 L 0 0 0 16" fill="none" stroke="#06B6D4" stroke-width="0.8" opacity="0.3"/>
+        <circle cx="8" cy="8" r="1.5" fill="#06B6D4" opacity="0.4"/>
+      </pattern>
+    </defs>
+
+    <circle cx="280" cy="300" r="160" fill="#06B6D4" opacity="0.05" filter="url(#glow)"/>
+    <circle cx="520" cy="280" r="170" fill="#EC4899" opacity="0.05" filter="url(#glow)"/>
+
+    <path d="M 140 420 L 220 420 L 260 460" stroke="#06B6D4" stroke-width="1.5" fill="none" opacity="0.3" stroke-dasharray="4 4"/>
+    <path d="M 660 180 L 580 180 L 540 140" stroke="#EC4899" stroke-width="1.5" fill="none" opacity="0.3" stroke-dasharray="4 4"/>
+
+    <g transform="translate(180, 190)">
+      <rect x="0" y="0" width="200" height="240" rx="16" fill="#000000" opacity="0.4"/>
+      <rect x="0" y="0" width="200" height="240" rx="16" fill="url(#leftNodeGrad)" stroke="#06B6D4" stroke-width="2" opacity="0.9"/>
+      <rect x="16" y="16" width="168" height="208" rx="8" fill="url(#fingerprintGrid)"/>
+      <path d="M 40 60 L 90 60 L 130 100 L 130 160 L 80 180" stroke="#06B6D4" stroke-width="1.5" fill="none" opacity="0.6"/>
+      <path d="M 90 60 L 70 120 L 140 120" stroke="#06B6D4" stroke-width="1.5" fill="none" stroke-dasharray="2 2" opacity="0.5"/>
+      <circle cx="40" cy="60" r="4" fill="#FFFFFF" filter="url(#glow-subtle)"/>
+      <circle cx="90" cy="60" r="3.5" fill="#06B6D4"/>
+      <circle cx="130" cy="100" r="3.5" fill="#06B6D4"/>
+      <circle cx="70" cy="120" r="3.5" fill="#EC4899"/>
+      <circle cx="140" cy="120" r="3.5" fill="#06B6D4"/>
+      <circle cx="130" cy="160" r="4" fill="#FFFFFF" filter="url(#glow-subtle)"/>
+      <circle cx="80" cy="180" r="3.5" fill="#06B6D4"/>
+      <path d="M 24 36 L 24 24 L 36 24" stroke="#06B6D4" stroke-width="2" fill="none"/>
+      <path d="M 176 36 L 176 24 L 164 24" stroke="#06B6D4" stroke-width="2" fill="none"/>
+      <path d="M 24 184 L 24 196 L 36 196" stroke="#06B6D4" stroke-width="2" fill="none"/>
+      <path d="M 176 184 L 176 196 L 164 196" stroke="#06B6D4" stroke-width="2" fill="none"/>
+    </g>
+
+    <path d="M 330 330 C 400 420, 440 180, 500 270" stroke="url(#bridgeGrad)" stroke-width="12" fill="none" opacity="0.3" filter="url(#glow)"/>
+    <path d="M 320 340 C 390 440, 430 170, 510 260" stroke="url(#bridgeGrad)" stroke-width="4" fill="none" stroke-dasharray="12 6 4 6" stroke-linecap="round" filter="url(#glow-subtle)"/>
+    <path d="M 350 290 C 410 230, 430 350, 480 300" stroke="#FFFFFF" stroke-width="1.5" fill="none" stroke-dasharray="6 6" opacity="0.7"/>
+
+    <circle cx="375" cy="355" r="3" fill="#06B6D4" filter="url(#glow-subtle)"/>
+    <circle cx="415" cy="310" r="4.5" fill="#FFFFFF" filter="url(#glow)"/>
+    <circle cx="445" cy="255" r="3" fill="#EC4899" filter="url(#glow-subtle)"/>
+    <rect x="470" y="270" width="5" height="5" transform="rotate(45 472.5 272.5)" fill="#06B6D4"/>
+
+    <g transform="translate(420, 150)">
+      <rect x="0" y="0" width="220" height="260" rx="16" fill="#000000" opacity="0.5"/>
+      <rect x="0" y="0" width="220" height="260" rx="16" fill="url(#rightNodeGrad)" stroke="#EC4899" stroke-width="2.5" opacity="0.95"/>
+      <rect x="14" y="14" width="192" height="232" rx="10" fill="#0F172A" stroke="#FFFFFF" stroke-width="1" stroke-opacity="0.2"/>
+      <g opacity="0.9">
+        <circle cx="150" cy="75" r="18" fill="#EC4899" filter="url(#glow-subtle)"/>
+        <circle cx="150" cy="75" r="10" fill="#FFFFFF"/>
+        <polygon points="40,190 100,100 160,190" fill="#06B6D4" opacity="0.4"/>
+        <polygon points="80,200 140,120 200,200" fill="#EC4899" opacity="0.6"/>
+        <polygon points="140,120 200,200 140,200" fill="#FFFFFF" opacity="0.15"/>
+        <line x1="14" y1="160" x2="206" y2="160" stroke="#06B6D4" stroke-width="1" opacity="0.4" stroke-dasharray="4 4"/>
+        <line x1="80" y1="14" x2="80" y2="246" stroke="#06B6D4" stroke-width="1" opacity="0.2" stroke-dasharray="4 4"/>
+      </g>
+      <rect x="-5" y="-5" width="10" height="10" fill="#FFFFFF" stroke="#EC4899" stroke-width="2"/>
+      <rect x="215" y="-5" width="10" height="10" fill="#FFFFFF" stroke="#EC4899" stroke-width="2"/>
+      <rect x="-5" y="255" width="10" height="10" fill="#FFFFFF" stroke="#EC4899" stroke-width="2"/>
+      <rect x="215" y="255" width="10" height="10" fill="#FFFFFF" stroke="#EC4899" stroke-width="2"/>
+    </g>
+
+    <circle cx="320" cy="340" r="6" fill="#06B6D4" stroke="#FFFFFF" stroke-width="2" filter="url(#glow-subtle)"/>
+    <circle cx="510" cy="260" r="6" fill="#EC4899" stroke="#FFFFFF" stroke-width="2" filter="url(#glow-subtle)"/>
+  </svg>
+</p>
+
+# image-bridge-toolkit
+
+[![PyPI version](https://img.shields.io/pypi/v/image-bridge-toolkit.svg)](https://pypi.org/project/image-bridge-toolkit/)
+[![License: MIT](https://img.shields.io/badge/Code_License-MIT-yellow.svg)](LICENSE)
+[![License: CC BY-NC-ND 4.0](https://img.shields.io/badge/Docs_License-CC_BY--NC--ND_4.0-lightgrey.svg)](https://creativecommons.org/licenses/by-nc-nd/4.0/)
+
+`image-bridge-toolkit` provides cache-backed perceptual image fingerprinting and cross-dataset bridge matching designed to handle large scale (50,000+ image) libraries. It finds identical or highly similar images across disparate folders while remaining resilient to scale shifts, aspect ratio changes, heavy compression, and intrusive watermarks.
+
+---
+
+## Key Features
+
+- **Per-Directory JSON Caching**: Generates and maintains lightweight `image_cache.json` files within each directory, keyed by filename and modification time (`mtime`).
+- **Scale-Invariant Waveform Approximations**: Uses 16-float 1D Discrete Cosine Transform (DCT) grayscale histogram vectors to ignore local watermark text spikes and compression artifacts.
+- **5-Tier Matching Funnel**:
+  1. **Tier 1 (Bitwise pHash Luma + Color)**: Generous 64-bit Hamming filtering to eliminate obvious non-matches.
+  2. **Tier 2 (Histogram Waveform)**: Euclidean distance on 16 low-frequency DCT coefficients to filter lighting distribution changes.
+  3. **Tier 3 (LAB Perceptual Metrics)**: Vector distance across `lab_value`, `lab_hue`, `lab_chroma`, and `lab_warmth`.
+  4. **Tier 4 (Spatial Layout Tie-Breaker)**: 16-point color spatial coordinates down-weighting image outer margins where signatures and borders reside.
+  5. **Tier 5 (Optional SSIM Direct Comparison)**: Full structural similarity pixel comparison on top candidate matches.
+- **Resumable & High Performance**: Output streams as `.jsonl` objects, allowing runs over huge original sets to resume instantly without re-processing.
+- **Zero-Dependency Fallback**: Runs with `OpenCV` if available for acceleration, or falls back to pure `NumPy` + `Pillow`.
+
+---
+
+## Installation
+
+Install via PyPI:
+
+```bash
+# Minimal installation (NumPy + Pillow)
+pip install image-bridge-toolkit
+
+# Recommended installation (includes OpenCV acceleration & progress bars)
+pip install "image-bridge-toolkit[all]"
+
+```
+
+For development:
+
+```bash
+git clone [https://github.com/akingdom/image_bridge_toolkit.git](https://github.com/akingdom/image_bridge_toolkit.git)
+cd image_bridge_toolkit
+pip install -e ".[all]"
+
+```
+
+---
+
+## Usage
+
+### 1. Build or Update Metadata Caches (`imgcache-build`)
+
+Walks a root directory, scanning all subdirectories (skipping any directory containing `.ignore_subdir`), and generates or updates `image_cache.json`.
+
+```bash
+# Build cache for a query dataset (e.g., 1,000 watermarked images)
+imgcache-build /path/to/thumbnails_set
+
+# Build cache for a target dataset (e.g., 50,000 original images)
+imgcache-build /path/to/originals_set
+
+```
+
+### 2. Match Datasets (`imgmatch-bridge`)
+
+Matches images from the query dataset against the target library, generating a bridge mapping file with confidence scores and top candidate lists.
+
+```bash
+imgmatch-bridge /path/to/thumbnails_set /path/to/originals_set \
+    -o bridge_map.jsonl \
+    --max-hamming 16 \
+    --candidate-threshold 0.03 \
+    --direct-compare
+
+```
+
+#### Command-Line Options
+
+| Flag | Default | Description |
+| --- | --- | --- |
+| `-o`, `--output` | `bridge_map.jsonl` | File path for output streaming results. |
+| `--max-hamming` | `16` | Maximum allowed Tier 1 pHash Hamming distance threshold. |
+| `--candidate-threshold` | `0.03` | Percentage band (e.g., 3%) to include close secondary matches. |
+| `--direct-compare` | `False` | Performs Tier 5 pixel-level SSIM re-ranking on candidates. |
+
+---
+
+## Cache Integrity & Updating
+
+* **`mtime` Matching**: A cached entry is preserved as long as the file's modification time matches the record in `image_cache.json`.
+* **Partial Recomputation**: If an existing cache entry is missing a newly added metric type (e.g., `hist_waveform`), only the missing calculation is executed; existing valid metrics are preserved.
+* **Corrupt Cache Handling**: Invalid or unparseable JSON files are automatically caught and rebuilt without interrupting directory traversal.
+* **Subdirectory Exclusion**: Placing a `.ignore_subdir` marker inside any directory causes the builder and matcher to ignore that folder and all nested subdirectories.
+
+---
+
+## License & Copyright
+
+* **Source Code (`src/`)**: Licensed under the [MIT License](https://www.google.com/search?q=LICENSE).
+* **Documentation & Branding**: Copyright © 2026 Andrew Kingdom. Licensed under the [Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International License (CC BY-NC-ND 4.0)](https://creativecommons.org/licenses/by-nc-nd/4.0/).
+* **Translation Permission**: Permission is explicitly granted to translate this documentation into other languages, provided that full attribution to Andrew Kingdom is maintained, a direct link to the original repository is included, and no non-linguistic modifications or structural alterations are made to the content.

image_bridge_toolkit-0.1.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+image_bridge_toolkit/__init__.py,sha256=clqQ3meBLXZjPDxFVhDswnf43A2vFsWHL9H4wGWzizw,199
+image_bridge_toolkit/_imaging_backend.py,sha256=h9lWrNWuGv0tgSgt3bzslgdZN67u9fDSA5ZFu8_fCMk,382
+image_bridge_toolkit/cache_builder.py,sha256=_iimAXMQ2YFheZ_OviHU_zGC1aHs-csxWRzOJGLhd2I,6087
+image_bridge_toolkit/cache_io.py,sha256=I04o5H05VDRUZBsfJIOTMDfRf1CT3ja_pS6Hjo8QYyI,1647
+image_bridge_toolkit/calculators.py,sha256=589ihMo3C_QdoF6xb252Rs7uRH1r2fI06KA1kdMV-hk,5742
+image_bridge_toolkit/constants.py,sha256=q4MjZwQdrMlPzkcTcsqgy5kvjkcUQh6F2ZfVOFnuhTA,793
+image_bridge_toolkit/dataset_loader.py,sha256=2d7ChU3tpoqFbZheCr7gH9wKatpRyAEneeuruYN2fnE,3322
+image_bridge_toolkit/logging_utils.py,sha256=MSDp3nwJ73eMVHCyRC2i0ISY8XEyWaR833Iwjcf_oC4,494
+image_bridge_toolkit/matcher.py,sha256=t7idIm0Uv0xZ61XNgBDuLmQgGeP5dvYF9MEYooJvtFs,9539
+image_bridge_toolkit/planner.py,sha256=p94pj0gR4qjwS26QhOgUsPjnPBjUIZI56nH8i9kY0eM,1469
+image_bridge_toolkit/scanner.py,sha256=EBf27Etp8xJK-KZihDRRmKKmjetCTE2eGOzKilqvnyE,2238
+image_bridge_toolkit/similarity.py,sha256=2cBGqqcU-Tldfh-UT54a1zitMUk92WSrc6LhYf1CskY,3495
+image_bridge_toolkit-0.1.0.dist-info/licenses/LICENSE.md,sha256=Uvr2imokjzok77N-IjLJFu2cyELzeBRyMomqfN1LKGE,2093
+image_bridge_toolkit-0.1.0.dist-info/METADATA,sha256=P5jHMsRrKyn50rjZb-W2J_iopJLehjfTJ3rfmS2jzAs,12669
+image_bridge_toolkit-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+image_bridge_toolkit-0.1.0.dist-info/entry_points.txt,sha256=yv1qlswe9wfWBGmyiOMzwQQIIwVArtFLHGRLg_Lwpjg,127
+image_bridge_toolkit-0.1.0.dist-info/top_level.txt,sha256=xiJOp4KY-WInli_rbZyrCMHI7eH4Cd0dsmjjckHJ-O4,21
+image_bridge_toolkit-0.1.0.dist-info/RECORD,,

image_bridge_toolkit-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

image_bridge_toolkit-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+imgcache-build = image_bridge_toolkit.cache_builder:main
+imgmatch-bridge = image_bridge_toolkit.matcher:main

image_bridge_toolkit-0.1.0.dist-info/licenses/LICENSE.md

@@ -0,0 +1,38 @@
+========================================================================
+SOURCE CODE LICENSE (MIT)
+========================================================================
+The source code within the `src/` directory is licensed under the MIT License.
+
+Copyright (c) 2026 Andrew Kingdom
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+========================================================================
+DOCUMENTATION & BRANDING LICENSE (CC BY-NC-ND 4.0 + TRANSLATION WAIVER)
+========================================================================
+All non-code assets, including README files, documentation, logo designs,
+and diagrams, are Copyright (c) 2026 Andrew Kingdom.
+
+Licensed under Creative Commons Attribution-NonCommercial-NoDerivatives 4.0
+International (CC BY-NC-ND 4.0).
+
+Additional Permission: Direct language translations of this documentation are
+expressly permitted without prior written authorization, provided that:
+1. Full attribution to Andrew Kingdom is maintained.
+2. A direct hyperlink to the original repository is included.
+3. No non-linguistic modifications or structural alterations are made.

image_bridge_toolkit-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+image_bridge_toolkit