imgsim-py 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- imgsim/__init__.py +49 -0
- imgsim/__main__.py +6 -0
- imgsim/_io.py +87 -0
- imgsim/_types.py +103 -0
- imgsim/cli.py +267 -0
- imgsim/cluster.py +72 -0
- imgsim/color.py +105 -0
- imgsim/embed.py +106 -0
- imgsim/fingerprint.py +133 -0
- imgsim/index.py +527 -0
- imgsim/py.typed +0 -0
- imgsim/report.py +138 -0
- imgsim_py-0.1.0.dist-info/METADATA +249 -0
- imgsim_py-0.1.0.dist-info/RECORD +18 -0
- imgsim_py-0.1.0.dist-info/WHEEL +5 -0
- imgsim_py-0.1.0.dist-info/entry_points.txt +2 -0
- imgsim_py-0.1.0.dist-info/licenses/LICENSE +21 -0
- imgsim_py-0.1.0.dist-info/top_level.txt +1 -0
imgsim/__init__.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""imgsim — find duplicate and near-duplicate images.
|
|
2
|
+
|
|
3
|
+
Quick start::
|
|
4
|
+
|
|
5
|
+
import imgsim
|
|
6
|
+
|
|
7
|
+
index = imgsim.ImageIndex()
|
|
8
|
+
index.add_dir("photos/", recursive=True)
|
|
9
|
+
result = index.compare(threshold=10)
|
|
10
|
+
|
|
11
|
+
for group in result.groups:
|
|
12
|
+
print(len(group), "similar images")
|
|
13
|
+
for path in group:
|
|
14
|
+
print(" ", path)
|
|
15
|
+
|
|
16
|
+
Why it exists: perceptual hashing alone produces two systematic false positives —
|
|
17
|
+
color-blind matches (a red apple and a green apple of the same shape share a
|
|
18
|
+
fingerprint) and collapsed fingerprints on flat images (a white rectangle and a
|
|
19
|
+
blue-white stripe pattern both hash to zero). imgsim adds a hue-aware color veto
|
|
20
|
+
and a low-information guard on top of the standard pHash + dHash pair.
|
|
21
|
+
|
|
22
|
+
Two algorithms are available:
|
|
23
|
+
|
|
24
|
+
``mode="hash"`` (default)
|
|
25
|
+
DCT perceptual hash + difference hash, 64 bits each, compared by Hamming
|
|
26
|
+
distance. No ML dependencies, ~3 s for 1000 images.
|
|
27
|
+
|
|
28
|
+
``mode="embed"``
|
|
29
|
+
ResNet18 embeddings compared by cosine similarity. Recognises the same subject
|
|
30
|
+
from different angles, at the cost of a PyTorch dependency.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
from ._types import Group, Pair, Result
|
|
36
|
+
from .index import ImageIndex
|
|
37
|
+
from .report import render_html, save_html
|
|
38
|
+
|
|
39
|
+
__version__ = "0.1.0"
|
|
40
|
+
|
|
41
|
+
__all__ = [
|
|
42
|
+
"Group",
|
|
43
|
+
"ImageIndex",
|
|
44
|
+
"Pair",
|
|
45
|
+
"Result",
|
|
46
|
+
"__version__",
|
|
47
|
+
"render_html",
|
|
48
|
+
"save_html",
|
|
49
|
+
]
|
imgsim/__main__.py
ADDED
imgsim/_io.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Image loading and preprocessing.
|
|
2
|
+
|
|
3
|
+
Every image is decoded exactly once, producing both the grayscale thumbnail used
|
|
4
|
+
for hashing and the small RGB grid used for color verification.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import contextlib
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
from PIL import Image, ImageOps
|
|
14
|
+
|
|
15
|
+
# 超大图(如全景照)默认会被 Pillow 拦截,这里解除限制,由调用方自行控制内存
|
|
16
|
+
Image.MAX_IMAGE_PIXELS = None
|
|
17
|
+
|
|
18
|
+
IMAGE_EXTENSIONS = frozenset(
|
|
19
|
+
{".jpg", ".jpeg", ".png", ".bmp", ".gif", ".webp", ".tif", ".tiff", ".avif", ".heic"}
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
_WHITE = (255, 255, 255, 255)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def iter_images(root: Path, recursive: bool = False) -> list[Path]:
|
|
26
|
+
"""列出目录下的图片文件,按路径排序以保证结果可复现。"""
|
|
27
|
+
it = root.rglob("*") if recursive else root.glob("*")
|
|
28
|
+
files = [p for p in it if p.is_file() and p.suffix.lower() in IMAGE_EXTENSIONS]
|
|
29
|
+
files.sort(key=lambda p: str(p).lower())
|
|
30
|
+
return files
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def load_features(path: Path, size: int, color_grid: int) -> tuple[np.ndarray, np.ndarray] | None:
|
|
34
|
+
"""Decode *path* once and return ``(gray, color_signature)``.
|
|
35
|
+
|
|
36
|
+
``gray`` is a ``size x size`` float64 array, ``color_signature`` is a
|
|
37
|
+
``color_grid * color_grid * 3`` float32 array with values in ``[0, 1]``.
|
|
38
|
+
|
|
39
|
+
Returns ``None`` when the file cannot be decoded (corrupt, not an image,
|
|
40
|
+
unreadable permission). Callers are expected to collect these paths rather
|
|
41
|
+
than abort the whole batch.
|
|
42
|
+
"""
|
|
43
|
+
try:
|
|
44
|
+
with Image.open(path) as im:
|
|
45
|
+
# JPEG only: let libjpeg downsample during decoding. A 4000x3000 photo
|
|
46
|
+
# decodes 5-10x faster because only 1/8 of the pixels are ever produced.
|
|
47
|
+
# No-op for other formats, and Pillow never scales below the requested size.
|
|
48
|
+
# Draft is a hint, not a guarantee — a failure here is not fatal.
|
|
49
|
+
with contextlib.suppress(Exception):
|
|
50
|
+
im.draft("RGB", (max(size, color_grid), max(size, color_grid)))
|
|
51
|
+
|
|
52
|
+
im = ImageOps.exif_transpose(im) # honour EXIF orientation before anything else
|
|
53
|
+
|
|
54
|
+
if im.mode in ("RGBA", "LA", "P"):
|
|
55
|
+
# Composite onto white: a transparent PNG would otherwise become
|
|
56
|
+
# solid black in grayscale and hash as if it were a dark photo.
|
|
57
|
+
im = im.convert("RGBA")
|
|
58
|
+
canvas = Image.new("RGBA", im.size, _WHITE)
|
|
59
|
+
canvas.alpha_composite(im)
|
|
60
|
+
im = canvas
|
|
61
|
+
im = im.convert("RGB")
|
|
62
|
+
|
|
63
|
+
# Region-average resampling is exactly the block mean we want for color signatures
|
|
64
|
+
signature = np.asarray(
|
|
65
|
+
im.resize((color_grid, color_grid), Image.BOX), dtype=np.float32
|
|
66
|
+
).ravel() / np.float32(255.0)
|
|
67
|
+
|
|
68
|
+
gray_im = im.convert("L")
|
|
69
|
+
if gray_im.size != (size, size):
|
|
70
|
+
gray_im = gray_im.resize((size, size), Image.LANCZOS)
|
|
71
|
+
gray = np.asarray(gray_im, dtype=np.float64)
|
|
72
|
+
|
|
73
|
+
return gray, signature
|
|
74
|
+
except Exception: # - a broken file must not kill a 100k-image batch
|
|
75
|
+
return None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def load_thumbnail(path: Path, max_side: int) -> Image.Image | None:
|
|
79
|
+
"""Load an RGB thumbnail for report rendering, preserving aspect ratio."""
|
|
80
|
+
try:
|
|
81
|
+
with Image.open(path) as im:
|
|
82
|
+
im.draft("RGB", (max_side, max_side))
|
|
83
|
+
im = ImageOps.exif_transpose(im).convert("RGB")
|
|
84
|
+
im.thumbnail((max_side, max_side), Image.LANCZOS)
|
|
85
|
+
return im.copy()
|
|
86
|
+
except Exception:
|
|
87
|
+
return None
|
imgsim/_types.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Public result types returned by :class:`imgsim.ImageIndex`."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
#: Maximum possible distance, used to normalise hash distances into a similarity score.
|
|
9
|
+
HASH_BITS = 64
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True)
|
|
13
|
+
class Pair:
|
|
14
|
+
"""A pair of images that the algorithm considers similar.
|
|
15
|
+
|
|
16
|
+
Attributes:
|
|
17
|
+
a: First image path.
|
|
18
|
+
b: Second image path.
|
|
19
|
+
distance: Raw distance. Hamming bits (0-64) in hash mode, cosine distance
|
|
20
|
+
(0-2) in embed mode. Lower is more similar.
|
|
21
|
+
kind: ``"hash"`` or ``"embed"``, indicating how the distance was measured.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
a: Path
|
|
25
|
+
b: Path
|
|
26
|
+
distance: float
|
|
27
|
+
kind: str = "hash"
|
|
28
|
+
|
|
29
|
+
@property
|
|
30
|
+
def similarity(self) -> float:
|
|
31
|
+
"""Distance normalised to ``[0, 1]``, where 1.0 means identical."""
|
|
32
|
+
if self.kind == "hash":
|
|
33
|
+
return max(0.0, 1.0 - self.distance / HASH_BITS)
|
|
34
|
+
return max(0.0, 1.0 - self.distance)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class Group:
|
|
39
|
+
"""A set of images that are transitively similar to each other.
|
|
40
|
+
|
|
41
|
+
``paths[0]`` is the group representative (the first image encountered); the
|
|
42
|
+
remaining paths are ordered by increasing distance to it.
|
|
43
|
+
|
|
44
|
+
Attributes:
|
|
45
|
+
paths: Image paths in this group, representative first.
|
|
46
|
+
distances: Distance of each path to ``paths[0]``. Always starts with 0.0.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
paths: list[Path]
|
|
50
|
+
distances: list[float] = field(default_factory=list)
|
|
51
|
+
|
|
52
|
+
def __len__(self) -> int:
|
|
53
|
+
return len(self.paths)
|
|
54
|
+
|
|
55
|
+
def __iter__(self):
|
|
56
|
+
return iter(self.paths)
|
|
57
|
+
|
|
58
|
+
def __getitem__(self, index):
|
|
59
|
+
return self.paths[index]
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def representative(self) -> Path:
|
|
63
|
+
"""The image this group is anchored on."""
|
|
64
|
+
return self.paths[0]
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def max_distance(self) -> float:
|
|
68
|
+
"""Largest distance within the group, i.e. how loose the group is."""
|
|
69
|
+
return max(self.distances) if self.distances else 0.0
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass
|
|
73
|
+
class Result:
|
|
74
|
+
"""Everything a single comparison produced.
|
|
75
|
+
|
|
76
|
+
Attributes:
|
|
77
|
+
groups: Similar groups, ordered by size (largest first).
|
|
78
|
+
pairs: Every similar pair, ordered by increasing distance.
|
|
79
|
+
n_images: Number of images successfully indexed.
|
|
80
|
+
failed: Paths that could not be decoded, so they were excluded.
|
|
81
|
+
threshold: The threshold used for this comparison.
|
|
82
|
+
kind: ``"hash"`` or ``"embed"``.
|
|
83
|
+
elapsed: Wall-clock seconds spent on the comparison (excluding indexing).
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
groups: list[Group] = field(default_factory=list)
|
|
87
|
+
pairs: list[Pair] = field(default_factory=list)
|
|
88
|
+
n_images: int = 0
|
|
89
|
+
failed: list[Path] = field(default_factory=list)
|
|
90
|
+
threshold: float = 10.0
|
|
91
|
+
kind: str = "hash"
|
|
92
|
+
elapsed: float = 0.0
|
|
93
|
+
|
|
94
|
+
@property
|
|
95
|
+
def duplicates(self) -> list[Group]:
|
|
96
|
+
"""Groups with exactly identical fingerprints (distance 0)."""
|
|
97
|
+
return [g for g in self.groups if g.max_distance == 0]
|
|
98
|
+
|
|
99
|
+
def __len__(self) -> int:
|
|
100
|
+
return len(self.groups)
|
|
101
|
+
|
|
102
|
+
def __iter__(self):
|
|
103
|
+
return iter(self.groups)
|
imgsim/cli.py
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
"""Command line interface.
|
|
2
|
+
|
|
3
|
+
Design notes:
|
|
4
|
+
* One positional argument (the directory) covers the common case; ``--query``
|
|
5
|
+
switches to nearest-neighbour lookup instead of a separate subcommand.
|
|
6
|
+
* ``--format json`` makes the tool composable in pipelines.
|
|
7
|
+
* ``--fail-on-duplicates`` lets CI reject a commit that introduces duplicate assets.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import csv
|
|
14
|
+
import json
|
|
15
|
+
import sys
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from . import __version__
|
|
19
|
+
from ._types import Result
|
|
20
|
+
from .index import (
|
|
21
|
+
DEFAULT_COLOR_THRESHOLD,
|
|
22
|
+
DEFAULT_EMBED_THRESHOLD,
|
|
23
|
+
DEFAULT_HASH_THRESHOLD,
|
|
24
|
+
ImageIndex,
|
|
25
|
+
)
|
|
26
|
+
from .report import save_html
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _is_tty() -> bool:
|
|
30
|
+
return hasattr(sys.stdout, "isatty") and sys.stdout.isatty()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class _Progress:
|
|
34
|
+
"""Single-line progress reporter that stays quiet when piped."""
|
|
35
|
+
|
|
36
|
+
def __init__(self, label: str, enabled: bool = True) -> None:
|
|
37
|
+
self.label = label
|
|
38
|
+
self.enabled = enabled and _is_tty()
|
|
39
|
+
self._started = False
|
|
40
|
+
|
|
41
|
+
def __call__(self, done: int, total: int) -> None:
|
|
42
|
+
if not self.enabled:
|
|
43
|
+
return
|
|
44
|
+
self._started = True
|
|
45
|
+
width = 24
|
|
46
|
+
filled = int(width * done / total) if total else width
|
|
47
|
+
bar = "=" * filled + " " * (width - filled)
|
|
48
|
+
print(f"\r {self.label} [{bar}] {done}/{total}", end="", flush=True)
|
|
49
|
+
|
|
50
|
+
def close(self) -> None:
|
|
51
|
+
if self._started:
|
|
52
|
+
print()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
56
|
+
parser = argparse.ArgumentParser(
|
|
57
|
+
prog="imgsim",
|
|
58
|
+
description="Find duplicate and near-duplicate images.",
|
|
59
|
+
epilog="Examples:\n"
|
|
60
|
+
" imgsim ./photos\n"
|
|
61
|
+
" imgsim ./photos -r -t 8 -o ./out\n"
|
|
62
|
+
" imgsim ./photos --query cat.jpg -k 5\n"
|
|
63
|
+
" imgsim ./photos --format json | jq '.groups | length'",
|
|
64
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
65
|
+
)
|
|
66
|
+
parser.add_argument("directory", type=Path, help="Directory containing images.")
|
|
67
|
+
parser.add_argument(
|
|
68
|
+
"-t", "--threshold", type=float, default=None,
|
|
69
|
+
help=f"Similarity cutoff. Hamming bits in hash mode (default {DEFAULT_HASH_THRESHOLD:g}), "
|
|
70
|
+
f"cosine distance in embed mode (default {DEFAULT_EMBED_THRESHOLD:g}). Lower is stricter.",
|
|
71
|
+
)
|
|
72
|
+
parser.add_argument(
|
|
73
|
+
"-r", "--recursive", action="store_true", help="Descend into subdirectories."
|
|
74
|
+
)
|
|
75
|
+
parser.add_argument(
|
|
76
|
+
"--mode", choices=("hash", "embed"), default="hash",
|
|
77
|
+
help="hash = perceptual fingerprint (default, no ML deps); embed = ResNet18 features.",
|
|
78
|
+
)
|
|
79
|
+
parser.add_argument(
|
|
80
|
+
"--color-threshold", type=float, default=DEFAULT_COLOR_THRESHOLD,
|
|
81
|
+
help=f"Color veto cutoff, hash mode only (default {DEFAULT_COLOR_THRESHOLD}). "
|
|
82
|
+
"Lower rejects more 'same shape, different color' matches.",
|
|
83
|
+
)
|
|
84
|
+
parser.add_argument("-o", "--out", type=Path, default=Path("imgsim-out"),
|
|
85
|
+
help="Output directory (default ./imgsim-out).")
|
|
86
|
+
parser.add_argument(
|
|
87
|
+
"-f", "--format", choices=("text", "json", "csv"), default="text",
|
|
88
|
+
help="Stdout format (default text).",
|
|
89
|
+
)
|
|
90
|
+
parser.add_argument("--no-report", action="store_true", help="Skip HTML report generation.")
|
|
91
|
+
parser.add_argument("--topk", type=int, default=0,
|
|
92
|
+
help="Also write each image's top-K nearest neighbours to topk.csv.")
|
|
93
|
+
parser.add_argument("--workers", type=int, default=None,
|
|
94
|
+
help="Decoding threads (default: min(8, cpu_count)).")
|
|
95
|
+
parser.add_argument("-q", "--quiet", action="store_true", help="Suppress progress output.")
|
|
96
|
+
parser.add_argument("--cache", type=Path, default=None,
|
|
97
|
+
help="Fingerprint cache file (default <out>/features.npz).")
|
|
98
|
+
parser.add_argument("--reuse", action="store_true",
|
|
99
|
+
help="Load the cache if present instead of decoding, so threshold "
|
|
100
|
+
"changes are instant.")
|
|
101
|
+
parser.add_argument("--query", type=Path, default=None,
|
|
102
|
+
help="Find images similar to this one instead of grouping a directory.")
|
|
103
|
+
parser.add_argument("-k", "--neighbours", type=int, default=5,
|
|
104
|
+
help="Number of results for --query (default 5).")
|
|
105
|
+
parser.add_argument("--fail-on-duplicates", action="store_true",
|
|
106
|
+
help="Exit with status 1 if any similar group is found.")
|
|
107
|
+
parser.add_argument("--version", action="version", version=f"imgsim {__version__}")
|
|
108
|
+
return parser
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _print_text(result: Result, out_dir: Path, report: Path | None) -> None:
|
|
112
|
+
if not result.groups:
|
|
113
|
+
print("\nNo similar images found.")
|
|
114
|
+
else:
|
|
115
|
+
print(f"\n{len(result.groups)} group(s), {len(result.pairs)} similar pair(s):\n")
|
|
116
|
+
for number, group in enumerate(result.groups[:20], 1):
|
|
117
|
+
head = f" [{number}] {len(group)} images"
|
|
118
|
+
span = f" (max distance {group.max_distance:g})"
|
|
119
|
+
print(head + span)
|
|
120
|
+
for path, distance in zip(group.paths[:6], group.distances[:6]):
|
|
121
|
+
print(f" {distance:>6g} {path}")
|
|
122
|
+
if len(group) > 6:
|
|
123
|
+
print(f" ... and {len(group) - 6} more")
|
|
124
|
+
if len(result.groups) > 20:
|
|
125
|
+
print(f"\n ... and {len(result.groups) - 20} more groups")
|
|
126
|
+
if report:
|
|
127
|
+
print(f"\nReport: {report}")
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _print_json(result: Result) -> None:
|
|
131
|
+
payload = {
|
|
132
|
+
"images": result.n_images,
|
|
133
|
+
"threshold": result.threshold,
|
|
134
|
+
"mode": result.kind,
|
|
135
|
+
"elapsed": round(result.elapsed, 3),
|
|
136
|
+
"failed": [str(p) for p in result.failed],
|
|
137
|
+
"groups": [
|
|
138
|
+
{
|
|
139
|
+
"size": len(group),
|
|
140
|
+
"max_distance": group.max_distance,
|
|
141
|
+
"images": [{"path": str(p), "distance": d}
|
|
142
|
+
for p, d in zip(group.paths, group.distances)],
|
|
143
|
+
}
|
|
144
|
+
for group in result.groups
|
|
145
|
+
],
|
|
146
|
+
"pairs": [
|
|
147
|
+
{"a": str(pair.a), "b": str(pair.b), "distance": pair.distance}
|
|
148
|
+
for pair in result.pairs
|
|
149
|
+
],
|
|
150
|
+
}
|
|
151
|
+
json.dump(payload, sys.stdout, ensure_ascii=False, indent=2)
|
|
152
|
+
sys.stdout.write("\n")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _print_csv(result: Result) -> None:
|
|
156
|
+
writer = csv.writer(sys.stdout)
|
|
157
|
+
writer.writerow(["group", "size", "distance", "path"])
|
|
158
|
+
for number, group in enumerate(result.groups, 1):
|
|
159
|
+
for path, distance in zip(group.paths, group.distances):
|
|
160
|
+
writer.writerow([number, len(group), round(distance, 4), str(path)])
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _write_topk(result: Result, index: ImageIndex, path: Path, k: int) -> None:
|
|
164
|
+
with open(path, "w", newline="", encoding="utf-8-sig") as handle:
|
|
165
|
+
writer = csv.writer(handle)
|
|
166
|
+
writer.writerow(["image", "rank", "neighbour", "distance"])
|
|
167
|
+
for source in index.paths:
|
|
168
|
+
for rank, (neighbour, distance) in enumerate(index.nearest(source, k), 1):
|
|
169
|
+
if neighbour == source:
|
|
170
|
+
continue
|
|
171
|
+
writer.writerow([source.name, rank, neighbour.name, round(distance, 4)])
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def main(argv: list[str] | None = None) -> int:
|
|
175
|
+
parser = _build_parser()
|
|
176
|
+
args = parser.parse_args(argv)
|
|
177
|
+
|
|
178
|
+
directory: Path = args.directory.expanduser()
|
|
179
|
+
if not directory.is_dir():
|
|
180
|
+
parser.error(f"not a directory: {directory}")
|
|
181
|
+
|
|
182
|
+
out_dir: Path = args.out.expanduser()
|
|
183
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
184
|
+
cache: Path = args.cache or (out_dir / "features.npz")
|
|
185
|
+
|
|
186
|
+
index = ImageIndex(mode=args.mode)
|
|
187
|
+
|
|
188
|
+
reused = False
|
|
189
|
+
if args.reuse and cache.exists():
|
|
190
|
+
try:
|
|
191
|
+
index = ImageIndex.load(cache)
|
|
192
|
+
if index.mode != args.mode:
|
|
193
|
+
index = ImageIndex(mode=args.mode)
|
|
194
|
+
else:
|
|
195
|
+
reused = True
|
|
196
|
+
except Exception as exc: # - a stale cache should never be fatal
|
|
197
|
+
print(f"warning: could not reuse cache ({exc}); re-indexing", file=sys.stderr)
|
|
198
|
+
index = ImageIndex(mode=args.mode)
|
|
199
|
+
|
|
200
|
+
if not reused:
|
|
201
|
+
progress = _Progress("indexing", enabled=not args.quiet)
|
|
202
|
+
try:
|
|
203
|
+
index.add_dir(directory, recursive=args.recursive,
|
|
204
|
+
workers=args.workers, progress=progress)
|
|
205
|
+
except RuntimeError as exc:
|
|
206
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
207
|
+
return 2
|
|
208
|
+
finally:
|
|
209
|
+
progress.close()
|
|
210
|
+
index.save(cache)
|
|
211
|
+
|
|
212
|
+
if args.query is not None:
|
|
213
|
+
neighbours = index.nearest(args.query, args.neighbours)
|
|
214
|
+
if args.format == "json":
|
|
215
|
+
json.dump(
|
|
216
|
+
[{"path": str(p), "distance": d} for p, d in neighbours],
|
|
217
|
+
sys.stdout, ensure_ascii=False, indent=2,
|
|
218
|
+
)
|
|
219
|
+
sys.stdout.write("\n")
|
|
220
|
+
else:
|
|
221
|
+
print(f"\n{len(neighbours)} nearest to {args.query.name}:")
|
|
222
|
+
for path, distance in neighbours:
|
|
223
|
+
print(f" {distance:>8.3f} {path}")
|
|
224
|
+
if index.failed:
|
|
225
|
+
print(f"\n{len(index.failed)} file(s) could not be decoded.", file=sys.stderr)
|
|
226
|
+
return 0
|
|
227
|
+
|
|
228
|
+
result = index.compare(threshold=args.threshold, color_threshold=args.color_threshold)
|
|
229
|
+
|
|
230
|
+
report: Path | None = None
|
|
231
|
+
if not args.no_report:
|
|
232
|
+
report = save_html(result, out_dir / "report.html")
|
|
233
|
+
|
|
234
|
+
if result.groups:
|
|
235
|
+
with open(out_dir / "groups.csv", "w", newline="", encoding="utf-8-sig") as handle:
|
|
236
|
+
writer = csv.writer(handle)
|
|
237
|
+
writer.writerow(["group", "size", "distance", "path"])
|
|
238
|
+
for number, group in enumerate(result.groups, 1):
|
|
239
|
+
for path, distance in zip(group.paths, group.distances):
|
|
240
|
+
writer.writerow([number, len(group), round(distance, 4), str(path)])
|
|
241
|
+
with open(out_dir / "pairs.csv", "w", newline="", encoding="utf-8-sig") as handle:
|
|
242
|
+
writer = csv.writer(handle)
|
|
243
|
+
writer.writerow(["a", "b", "distance", "similarity"])
|
|
244
|
+
for pair in result.pairs:
|
|
245
|
+
writer.writerow([str(pair.a), str(pair.b),
|
|
246
|
+
round(pair.distance, 4), round(pair.similarity, 4)])
|
|
247
|
+
|
|
248
|
+
if args.topk > 0:
|
|
249
|
+
_write_topk(result, index, out_dir / "topk.csv", args.topk)
|
|
250
|
+
|
|
251
|
+
if args.format == "json":
|
|
252
|
+
_print_json(result)
|
|
253
|
+
elif args.format == "csv":
|
|
254
|
+
_print_csv(result)
|
|
255
|
+
else:
|
|
256
|
+
_print_text(result, out_dir, report)
|
|
257
|
+
|
|
258
|
+
if result.failed:
|
|
259
|
+
print(f"{len(result.failed)} file(s) could not be decoded.", file=sys.stderr)
|
|
260
|
+
|
|
261
|
+
if args.fail_on_duplicates and result.groups:
|
|
262
|
+
return 1
|
|
263
|
+
return 0
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
if __name__ == "__main__":
|
|
267
|
+
raise SystemExit(main())
|
imgsim/cluster.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Transitive grouping of similar images.
|
|
2
|
+
|
|
3
|
+
Similarity is not transitive — A looks like B, B looks like C, but A and C may be
|
|
4
|
+
far apart. Treating it as transitive anyway (union-find) is the standard choice for
|
|
5
|
+
deduplication: it produces groups where every image is reachable from every other
|
|
6
|
+
through a chain of similar pairs, which is what a user means by "these are the same
|
|
7
|
+
photo". The cost is occasional chaining on long gradient series, which callers can
|
|
8
|
+
bound by lowering the threshold.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class DisjointSet:
|
|
17
|
+
"""Union-find with path halving and union by size."""
|
|
18
|
+
|
|
19
|
+
__slots__ = ("_parent", "_size")
|
|
20
|
+
|
|
21
|
+
def __init__(self, n: int) -> None:
|
|
22
|
+
self._parent = list(range(n))
|
|
23
|
+
self._size = [1] * n
|
|
24
|
+
|
|
25
|
+
def find(self, x: int) -> int:
|
|
26
|
+
parent = self._parent
|
|
27
|
+
while parent[x] != x:
|
|
28
|
+
parent[x] = parent[parent[x]] # path halving
|
|
29
|
+
x = parent[x]
|
|
30
|
+
return x
|
|
31
|
+
|
|
32
|
+
def union(self, a: int, b: int) -> bool:
|
|
33
|
+
"""Merge the sets containing *a* and *b*. Returns True if they were separate."""
|
|
34
|
+
ra, rb = self.find(a), self.find(b)
|
|
35
|
+
if ra == rb:
|
|
36
|
+
return False
|
|
37
|
+
if self._size[ra] < self._size[rb]:
|
|
38
|
+
ra, rb = rb, ra
|
|
39
|
+
self._parent[rb] = ra
|
|
40
|
+
self._size[ra] += self._size[rb]
|
|
41
|
+
return True
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def connected_groups(distances: np.ndarray, threshold: float) -> list[list[int]]:
|
|
45
|
+
"""Group indices whose pairwise distance is at or below *threshold*.
|
|
46
|
+
|
|
47
|
+
Args:
|
|
48
|
+
distances: Square symmetric distance matrix.
|
|
49
|
+
threshold: Inclusive cutoff.
|
|
50
|
+
|
|
51
|
+
Returns:
|
|
52
|
+
Index lists for groups of size >= 2, largest group first. Images with no
|
|
53
|
+
neighbour are omitted — callers usually only care about what clustered.
|
|
54
|
+
"""
|
|
55
|
+
n = distances.shape[0]
|
|
56
|
+
if n < 2:
|
|
57
|
+
return []
|
|
58
|
+
|
|
59
|
+
rows, cols = np.triu_indices(n, k=1)
|
|
60
|
+
close = distances[rows, cols] <= threshold
|
|
61
|
+
|
|
62
|
+
forest = DisjointSet(n)
|
|
63
|
+
for i, j in zip(rows[close].tolist(), cols[close].tolist()):
|
|
64
|
+
forest.union(i, j)
|
|
65
|
+
|
|
66
|
+
buckets: dict[int, list[int]] = {}
|
|
67
|
+
for i in range(n):
|
|
68
|
+
buckets.setdefault(forest.find(i), []).append(i)
|
|
69
|
+
|
|
70
|
+
groups = [members for members in buckets.values() if len(members) > 1]
|
|
71
|
+
groups.sort(key=lambda g: (-len(g), g[0]))
|
|
72
|
+
return groups
|
imgsim/color.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Color verification on top of perceptual hashing.
|
|
2
|
+
|
|
3
|
+
Perceptual hashes are computed on grayscale pixels, which means a red apple and a
|
|
4
|
+
green apple of the same shape produce nearly identical fingerprints. This module
|
|
5
|
+
adds a cheap color signature that vetoes such matches.
|
|
6
|
+
|
|
7
|
+
The signature is a small RGB grid (8x8 by default) taken by region averaging. The
|
|
8
|
+
distance between two signatures is computed in HSL space rather than RGB, for one
|
|
9
|
+
specific reason: converting an image to grayscale destroys its saturation but not
|
|
10
|
+
its perceived lightness. If we compared RGB directly, "photo vs its black-and-white
|
|
11
|
+
version" would look as different as "red apple vs green apple" — both are large
|
|
12
|
+
color changes. In HSL, the hue and saturation terms are weighted by the *smaller*
|
|
13
|
+
saturation of the two images, so they automatically vanish for grayscale images
|
|
14
|
+
and only lightness is compared. Hue differences between two saturated images
|
|
15
|
+
still register at full strength.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
|
|
22
|
+
# Weights were calibrated on the bundled test corpus (see tests/test_color.py):
|
|
23
|
+
# same-image variants score 0.002-0.06, unrelated images score 0.39-0.90.
|
|
24
|
+
HUE_WEIGHT = 0.55
|
|
25
|
+
SATURATION_WEIGHT = 0.35
|
|
26
|
+
LIGHTNESS_WEIGHT = 0.90
|
|
27
|
+
|
|
28
|
+
# Fraction of grid cells considered when scoring. Using the most-different cells
|
|
29
|
+
# instead of the mean keeps localized changes visible (recoloring one object
|
|
30
|
+
# affects only a few cells and would be averaged away).
|
|
31
|
+
DEFAULT_TOP_K = 12
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def rgb_to_hsl(signature: np.ndarray) -> np.ndarray:
|
|
35
|
+
"""Convert ``(n, cells * 3)`` RGB in ``[0, 1]`` to ``(n, cells, 3)`` HSL.
|
|
36
|
+
|
|
37
|
+
Lightness is the perceptual ``0.299 R + 0.587 G + 0.114 B``, not the HSV value
|
|
38
|
+
(``max(r, g, b)``). This matters: pure red has value 1.0 but lightness 0.299,
|
|
39
|
+
and its grayscale conversion also has lightness 0.299 — so the two agree, while
|
|
40
|
+
using HSV value they would differ by 0.7.
|
|
41
|
+
"""
|
|
42
|
+
px = np.ascontiguousarray(signature, dtype=np.float32).reshape(signature.shape[0], -1, 3)
|
|
43
|
+
r, g, b = px[..., 0], px[..., 1], px[..., 2]
|
|
44
|
+
hi, lo = px.max(-1), px.min(-1)
|
|
45
|
+
delta = hi - lo
|
|
46
|
+
|
|
47
|
+
saturation = np.where(hi > 1e-6, delta / np.maximum(hi, 1e-6), 0.0)
|
|
48
|
+
|
|
49
|
+
hue = np.zeros_like(hi)
|
|
50
|
+
chromatic = delta > 1e-6
|
|
51
|
+
idx = chromatic & (hi == r)
|
|
52
|
+
hue[idx] = ((g - b)[idx] / delta[idx]) % 6.0
|
|
53
|
+
idx = chromatic & (hi == g)
|
|
54
|
+
hue[idx] = ((b - r)[idx] / delta[idx]) + 2.0
|
|
55
|
+
idx = chromatic & (hi == b)
|
|
56
|
+
hue[idx] = ((r - g)[idx] / delta[idx]) + 4.0
|
|
57
|
+
hue /= 6.0
|
|
58
|
+
|
|
59
|
+
lightness = 0.299 * r + 0.587 * g + 0.114 * b
|
|
60
|
+
return np.stack([hue, saturation, lightness], axis=-1).astype(np.float32)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def color_distance(
|
|
64
|
+
a: np.ndarray,
|
|
65
|
+
b: np.ndarray,
|
|
66
|
+
top_k: int = DEFAULT_TOP_K,
|
|
67
|
+
chunk: int = 128,
|
|
68
|
+
) -> np.ndarray:
|
|
69
|
+
"""Color distance matrix between two sets of signatures.
|
|
70
|
+
|
|
71
|
+
Args:
|
|
72
|
+
a: ``(m, cells * 3)`` signatures.
|
|
73
|
+
b: ``(n, cells * 3)`` signatures.
|
|
74
|
+
top_k: How many of the most-different cells to average over.
|
|
75
|
+
chunk: Row chunk size, bounding peak memory to ``chunk * n * cells`` floats.
|
|
76
|
+
|
|
77
|
+
Returns:
|
|
78
|
+
``(m, n)`` float32 matrix. Values below ~0.18 indicate the two images share
|
|
79
|
+
a color layout; above that they are probably different subjects.
|
|
80
|
+
"""
|
|
81
|
+
n = b.shape[0]
|
|
82
|
+
a_hsl = rgb_to_hsl(a)
|
|
83
|
+
b_hsl = rgb_to_hsl(b)
|
|
84
|
+
out = np.empty((a_hsl.shape[0], n), dtype=np.float32)
|
|
85
|
+
|
|
86
|
+
for start in range(0, a_hsl.shape[0], chunk):
|
|
87
|
+
stop = min(start + chunk, a_hsl.shape[0])
|
|
88
|
+
left = a_hsl[start:stop, None, :, :]
|
|
89
|
+
right = b_hsl[None, :, :, :]
|
|
90
|
+
|
|
91
|
+
hue_delta = np.abs(left[..., 0] - right[..., 0])
|
|
92
|
+
hue_delta = np.minimum(hue_delta, 1.0 - hue_delta) * 2.0 # hue is circular
|
|
93
|
+
sat_delta = np.abs(left[..., 1] - right[..., 1])
|
|
94
|
+
light_delta = np.abs(left[..., 2] - right[..., 2])
|
|
95
|
+
sat_floor = np.minimum(left[..., 1], right[..., 1])
|
|
96
|
+
|
|
97
|
+
total = (
|
|
98
|
+
HUE_WEIGHT * hue_delta * sat_floor
|
|
99
|
+
+ SATURATION_WEIGHT * sat_delta * sat_floor
|
|
100
|
+
+ LIGHTNESS_WEIGHT * light_delta
|
|
101
|
+
)
|
|
102
|
+
k = min(top_k, total.shape[2])
|
|
103
|
+
out[start:stop] = np.partition(total, -k, axis=2)[:, :, -k:].mean(axis=2)
|
|
104
|
+
|
|
105
|
+
return out
|