easyannotate 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_factory/__init__.py +82 -0
- data_factory/__main__.py +6 -0
- data_factory/annotate/__init__.py +15 -0
- data_factory/annotate/attach.py +398 -0
- data_factory/annotate/demo.py +161 -0
- data_factory/annotate/images.py +443 -0
- data_factory/annotate/labels.py +399 -0
- data_factory/annotate/loop.py +359 -0
- data_factory/annotate/projects.py +207 -0
- data_factory/annotate/review.py +471 -0
- data_factory/annotate/schema.sql +171 -0
- data_factory/annotate/shapes.py +488 -0
- data_factory/annotate/store.py +292 -0
- data_factory/annotate/tools/__init__.py +39 -0
- data_factory/annotate/tools/fake.py +170 -0
- data_factory/annotate/tools/install.py +254 -0
- data_factory/annotate/tools/local.py +389 -0
- data_factory/annotate/tools/postprocess.py +175 -0
- data_factory/annotate/tools/protocol.py +238 -0
- data_factory/annotate/tools/registry.py +196 -0
- data_factory/annotate/tools/results.py +83 -0
- data_factory/annotate/tools/serve.py +106 -0
- data_factory/cli/__init__.py +21 -0
- data_factory/cli/main.py +837 -0
- data_factory/cli/menu.py +173 -0
- data_factory/cli/render.py +158 -0
- data_factory/core/__init__.py +11 -0
- data_factory/core/annotations/__init__.py +5 -0
- data_factory/core/annotations/coco.py +394 -0
- data_factory/core/annotations/csvfmt.py +174 -0
- data_factory/core/annotations/dota.py +98 -0
- data_factory/core/annotations/formats.py +626 -0
- data_factory/core/annotations/labelme.py +153 -0
- data_factory/core/annotations/model.py +334 -0
- data_factory/core/annotations/voc.py +231 -0
- data_factory/core/annotations/yolo.py +283 -0
- data_factory/core/bktree.py +89 -0
- data_factory/core/fs.py +245 -0
- data_factory/core/imaging.py +626 -0
- data_factory/core/ops/__init__.py +44 -0
- data_factory/core/ops/analyze.py +383 -0
- data_factory/core/ops/annotate_export.py +326 -0
- data_factory/core/ops/audit.py +274 -0
- data_factory/core/ops/augment.py +454 -0
- data_factory/core/ops/collect.py +241 -0
- data_factory/core/ops/combine.py +582 -0
- data_factory/core/ops/convert.py +212 -0
- data_factory/core/ops/dataset.py +246 -0
- data_factory/core/ops/dedup.py +235 -0
- data_factory/core/ops/export.py +168 -0
- data_factory/core/ops/image.py +407 -0
- data_factory/core/ops/labels.py +215 -0
- data_factory/core/ops/pairing.py +133 -0
- data_factory/core/ops/rename.py +154 -0
- data_factory/core/ops/report.py +323 -0
- data_factory/core/ops/visualize.py +164 -0
- data_factory/core/pairs.py +144 -0
- data_factory/core/parallel.py +191 -0
- data_factory/core/provenance.py +136 -0
- data_factory/core/registry.py +339 -0
- data_factory/core/stats.py +403 -0
- data_factory/core/types.py +148 -0
- data_factory/core/writer.py +194 -0
- data_factory/desktop.py +207 -0
- data_factory/hardware.py +243 -0
- data_factory/pipeline.py +192 -0
- data_factory/py.typed +0 -0
- data_factory/service/__init__.py +26 -0
- data_factory/service/app.py +483 -0
- data_factory/service/binding.py +174 -0
- data_factory/service/deps.py +133 -0
- data_factory/service/jobs.py +177 -0
- data_factory/service/limits.py +71 -0
- data_factory/service/routers/__init__.py +11 -0
- data_factory/service/routers/annotate.py +582 -0
- data_factory/service/routers/local.py +464 -0
- data_factory/service/routers/media.py +154 -0
- data_factory/service/routers/predictions.py +116 -0
- data_factory/service/routers/promote.py +182 -0
- data_factory/service/routers/tools.py +308 -0
- data_factory/service/static/annotate/api.js +246 -0
- data_factory/service/static/annotate/app.js +1696 -0
- data_factory/service/static/annotate/assist.js +106 -0
- data_factory/service/static/annotate/filmstrip.js +286 -0
- data_factory/service/static/annotate/hit.js +146 -0
- data_factory/service/static/annotate/labels.js +197 -0
- data_factory/service/static/annotate/render.js +286 -0
- data_factory/service/static/annotate/scene.js +277 -0
- data_factory/service/static/annotate/shapelist.js +72 -0
- data_factory/service/static/annotate/tools/box.js +96 -0
- data_factory/service/static/annotate/tools/line.js +96 -0
- data_factory/service/static/annotate/tools/obb.js +88 -0
- data_factory/service/static/annotate/tools/point.js +49 -0
- data_factory/service/static/annotate/tools/polygon.js +110 -0
- data_factory/service/static/annotate/tools/select.js +227 -0
- data_factory/service/static/annotate/undo.js +176 -0
- data_factory/service/static/annotate/viewport.js +156 -0
- data_factory/service/static/annotate.html +390 -0
- data_factory/service/static/index.html +632 -0
- data_factory/service/workspace.py +244 -0
- easyannotate/__init__.py +64 -0
- easyannotate/__main__.py +5 -0
- easyannotate/project.py +181 -0
- easyannotate/py.typed +0 -0
- easyannotate-0.4.0.dist-info/METADATA +767 -0
- easyannotate-0.4.0.dist-info/RECORD +109 -0
- easyannotate-0.4.0.dist-info/WHEEL +4 -0
- easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
- easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
"""Dataset health, and train/validation leakage.
|
|
2
|
+
|
|
3
|
+
``inspect.dataset`` answers "is this folder well-formed". These two answer the
|
|
4
|
+
harder question: "will training on this produce a model whose score means
|
|
5
|
+
anything".
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Callable
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from .. import fs, parallel, provenance, stats
|
|
15
|
+
from ..annotations import formats
|
|
16
|
+
from ..annotations.model import AnnotationError
|
|
17
|
+
from ..bktree import BKTree
|
|
18
|
+
from ..registry import Param, operation
|
|
19
|
+
from ..types import Context, OpResult
|
|
20
|
+
|
|
21
|
+
HEALTH_PARAMS = (
|
|
22
|
+
Param("folder", "dir", "Dataset to assess.", required=True, must_exist=True),
|
|
23
|
+
Param(
|
|
24
|
+
"source_format",
|
|
25
|
+
"choice",
|
|
26
|
+
"Input format; 'auto' sniffs the layout.",
|
|
27
|
+
default="auto",
|
|
28
|
+
choices=("auto",) + formats.FORMATS,
|
|
29
|
+
),
|
|
30
|
+
Param("recursive", "bool", "Include sub-folders.", default=True),
|
|
31
|
+
Param("top_classes", "int", "How many classes to list in the summary.", default=25, minimum=1),
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@operation(
|
|
36
|
+
"inspect.health",
|
|
37
|
+
title="Dataset health check",
|
|
38
|
+
category="inspect",
|
|
39
|
+
summary="Class balance, object sizes, geometry and composition -- with ranked findings and fixes.",
|
|
40
|
+
params=HEALTH_PARAMS,
|
|
41
|
+
notes=(
|
|
42
|
+
"Read-only, and works on any supported format.\n"
|
|
43
|
+
"\n"
|
|
44
|
+
"What it is actually checking for, and why each matters:\n"
|
|
45
|
+
" class imbalance a 50x gap means the rare class is barely learned\n"
|
|
46
|
+
" tiny objects under ~0.05% of the frame, they vanish on resize\n"
|
|
47
|
+
" case-variant classes 'Open' and 'open' train as two separate classes\n"
|
|
48
|
+
" zero-area boxes crash some trainers, are ignored by others\n"
|
|
49
|
+
" background share too many object-free images biases toward silence\n"
|
|
50
|
+
" mixed resolutions usually harmless, occasionally a sign of trouble\n"
|
|
51
|
+
"\n"
|
|
52
|
+
"Every threshold is a named constant in core/stats.py with its reasoning\n"
|
|
53
|
+
"written down -- they are judgement calls, not laws, and you should\n"
|
|
54
|
+
"disagree with them when your problem warrants it."
|
|
55
|
+
),
|
|
56
|
+
)
|
|
57
|
+
def health(params: dict[str, Any], ctx: Context) -> OpResult:
|
|
58
|
+
result = OpResult(op="inspect.health", dry_run=ctx.dry_run)
|
|
59
|
+
folder: Path = params["folder"]
|
|
60
|
+
|
|
61
|
+
try:
|
|
62
|
+
dataset = formats.read_dataset(
|
|
63
|
+
folder, params["source_format"], recursive=params["recursive"], ctx=ctx
|
|
64
|
+
)
|
|
65
|
+
except AnnotationError as exc:
|
|
66
|
+
result.fail(str(exc))
|
|
67
|
+
return result
|
|
68
|
+
|
|
69
|
+
ctx.emit(1, 1, "analysing")
|
|
70
|
+
payload = stats.analyse(dataset)
|
|
71
|
+
totals = payload["totals"]
|
|
72
|
+
|
|
73
|
+
result.counts.update(
|
|
74
|
+
{
|
|
75
|
+
"images": totals["images"],
|
|
76
|
+
"objects": totals["objects"],
|
|
77
|
+
"classes": totals["classes"],
|
|
78
|
+
"backgrounds": totals["backgrounds"],
|
|
79
|
+
"findings": len(payload["findings"]),
|
|
80
|
+
"errors": sum(1 for f in payload["findings"] if f["severity"] == "error"),
|
|
81
|
+
"warnings_found": sum(1 for f in payload["findings"] if f["severity"] == "warning"),
|
|
82
|
+
}
|
|
83
|
+
)
|
|
84
|
+
result.warnings.extend(dataset.warnings[:10])
|
|
85
|
+
result.data.update(
|
|
86
|
+
{
|
|
87
|
+
"format": dataset.source_format,
|
|
88
|
+
"findings": payload["findings"],
|
|
89
|
+
"per_class": payload["per_class"][: params["top_classes"]],
|
|
90
|
+
"balance": payload["balance"],
|
|
91
|
+
"object_sizes": payload["object_sizes"],
|
|
92
|
+
"relative_area": payload["relative_area"],
|
|
93
|
+
"objects_per_image": payload["objects_per_image"],
|
|
94
|
+
"image_sizes": payload["image_sizes"],
|
|
95
|
+
"splits": payload["splits"],
|
|
96
|
+
"geometry": payload["geometry"],
|
|
97
|
+
}
|
|
98
|
+
)
|
|
99
|
+
return result
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
LEAKAGE_PARAMS = (
|
|
103
|
+
Param("folder", "dir", "Split dataset to check (containing train/, val/, ...).",
|
|
104
|
+
required=True, must_exist=True),
|
|
105
|
+
Param("splits", "strs", "Split folder names (blank = every top-level folder).", default=None),
|
|
106
|
+
Param(
|
|
107
|
+
"method",
|
|
108
|
+
"choice",
|
|
109
|
+
"exact compares file bytes; phash and friends compare appearance.",
|
|
110
|
+
default="phash",
|
|
111
|
+
choices=("exact", "phash", "dhash", "ahash"),
|
|
112
|
+
),
|
|
113
|
+
Param(
|
|
114
|
+
"threshold",
|
|
115
|
+
"int",
|
|
116
|
+
"Perceptual distance counted as the same image (0 = identical).",
|
|
117
|
+
default=6,
|
|
118
|
+
minimum=0,
|
|
119
|
+
maximum=32,
|
|
120
|
+
),
|
|
121
|
+
Param("hash_size", "int", "Perceptual hash size.", default=8, minimum=4, maximum=32),
|
|
122
|
+
Param("max_examples", "int", "How many leaked pairs to list.", default=50, minimum=1),
|
|
123
|
+
Param("workers", "int", parallel.WORKERS_HELP, default=0, minimum=0, maximum=32),
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@operation(
|
|
128
|
+
"inspect.leakage",
|
|
129
|
+
title="Train/validation leakage",
|
|
130
|
+
category="inspect",
|
|
131
|
+
summary="Find near-duplicate images shared between splits, which silently inflate your score.",
|
|
132
|
+
params=LEAKAGE_PARAMS,
|
|
133
|
+
notes=(
|
|
134
|
+
"The most expensive bug in applied computer vision, and the hardest to\n"
|
|
135
|
+
"see: if the same photo -- or a burst frame taken a tenth of a second\n"
|
|
136
|
+
"apart, or a re-saved copy -- appears in both train and val, the model\n"
|
|
137
|
+
"has memorised the answer and your validation score is measuring recall\n"
|
|
138
|
+
"of the training set. Scores look excellent right up until deployment.\n"
|
|
139
|
+
"\n"
|
|
140
|
+
"It happens by accident constantly: video frames split randomly, a\n"
|
|
141
|
+
"dataset merged with an older copy of itself, images duplicated during\n"
|
|
142
|
+
"collection and then split.\n"
|
|
143
|
+
"\n"
|
|
144
|
+
"Read-only -- it reports, it never deletes. The fix is to re-split with\n"
|
|
145
|
+
"the duplicates grouped (dedup first with clean.dedup, then split), not\n"
|
|
146
|
+
"to remove the val copy and keep going.\n"
|
|
147
|
+
"\n"
|
|
148
|
+
"method=exact only catches byte-identical files; the default phash also\n"
|
|
149
|
+
"catches re-encoded, resized and lightly-cropped copies, which is where\n"
|
|
150
|
+
"most real leakage lives."
|
|
151
|
+
),
|
|
152
|
+
)
|
|
153
|
+
def leakage(params: dict[str, Any], ctx: Context) -> OpResult:
|
|
154
|
+
result = OpResult(op="inspect.leakage", dry_run=ctx.dry_run)
|
|
155
|
+
folder: Path = params["folder"]
|
|
156
|
+
|
|
157
|
+
groups = _split_groups(folder, params["splits"])
|
|
158
|
+
if len(groups) < 2:
|
|
159
|
+
result.fail(
|
|
160
|
+
f"need at least two splits to compare, found {len(groups)} under {folder}. "
|
|
161
|
+
f"Point this at the parent folder holding train/ and val/, or name them "
|
|
162
|
+
f"with --splits train,val"
|
|
163
|
+
)
|
|
164
|
+
return result
|
|
165
|
+
|
|
166
|
+
method = params["method"]
|
|
167
|
+
try:
|
|
168
|
+
fingerprint = _fingerprinter(method, params["hash_size"])
|
|
169
|
+
except RuntimeError as exc:
|
|
170
|
+
result.fail(str(exc))
|
|
171
|
+
return result
|
|
172
|
+
|
|
173
|
+
total = sum(len(paths) for paths in groups.values())
|
|
174
|
+
hashes: dict[str, list[tuple[Any, Path]]] = {}
|
|
175
|
+
unreadable = 0
|
|
176
|
+
|
|
177
|
+
def note_failure(path: Path, exc: Exception) -> None:
|
|
178
|
+
nonlocal unreadable
|
|
179
|
+
unreadable += 1
|
|
180
|
+
if unreadable <= 5:
|
|
181
|
+
result.warnings.append(f"{path.name}: unreadable ({exc})")
|
|
182
|
+
|
|
183
|
+
for split, paths in groups.items():
|
|
184
|
+
hashes[split] = [
|
|
185
|
+
(digest, path)
|
|
186
|
+
for path, digest in parallel.map_ordered(
|
|
187
|
+
paths,
|
|
188
|
+
fingerprint,
|
|
189
|
+
ctx=ctx,
|
|
190
|
+
workers=params["workers"],
|
|
191
|
+
label=lambda path, split=split: f"{split}/{path.name}",
|
|
192
|
+
on_error=note_failure,
|
|
193
|
+
)
|
|
194
|
+
if digest is not None
|
|
195
|
+
]
|
|
196
|
+
|
|
197
|
+
names = sorted(groups)
|
|
198
|
+
matches: list[dict[str, Any]] = []
|
|
199
|
+
per_pair: dict[str, int] = {}
|
|
200
|
+
leaked_files: set[Path] = set()
|
|
201
|
+
|
|
202
|
+
for i, left in enumerate(names):
|
|
203
|
+
for right in names[i + 1 :]:
|
|
204
|
+
found = (
|
|
205
|
+
_exact_matches(hashes[left], hashes[right])
|
|
206
|
+
if method == "exact"
|
|
207
|
+
else _near_matches(hashes[left], hashes[right], params["threshold"])
|
|
208
|
+
)
|
|
209
|
+
per_pair[f"{left} <-> {right}"] = len(found)
|
|
210
|
+
for distance, a, b in found:
|
|
211
|
+
leaked_files.add(b)
|
|
212
|
+
if len(matches) < params["max_examples"]:
|
|
213
|
+
matches.append(
|
|
214
|
+
{
|
|
215
|
+
"distance": distance,
|
|
216
|
+
"splits": f"{left} <-> {right}",
|
|
217
|
+
left: str(a),
|
|
218
|
+
right: str(b),
|
|
219
|
+
}
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
leaked = sum(per_pair.values())
|
|
223
|
+
smallest = min((len(v) for v in groups.values()), default=1)
|
|
224
|
+
result.counts.update(
|
|
225
|
+
{
|
|
226
|
+
"images": total,
|
|
227
|
+
"splits": len(groups),
|
|
228
|
+
"leaked_pairs": leaked,
|
|
229
|
+
"affected_images": len(leaked_files),
|
|
230
|
+
"unreadable": unreadable,
|
|
231
|
+
}
|
|
232
|
+
)
|
|
233
|
+
result.data.update(
|
|
234
|
+
{
|
|
235
|
+
"method": method,
|
|
236
|
+
"threshold": params["threshold"] if method != "exact" else 0,
|
|
237
|
+
"per_split": {k: len(v) for k, v in groups.items()},
|
|
238
|
+
"per_pair": per_pair,
|
|
239
|
+
"examples": matches,
|
|
240
|
+
}
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
if leaked:
|
|
244
|
+
share = len(leaked_files) / max(1, smallest)
|
|
245
|
+
result.warnings.append(
|
|
246
|
+
f"{leaked} leaked pair(s) across splits, touching {len(leaked_files)} image(s) "
|
|
247
|
+
f"-- around {share:.0%} of the smallest split. Validation scores from this "
|
|
248
|
+
f"dataset are optimistic."
|
|
249
|
+
)
|
|
250
|
+
result.warnings.append(
|
|
251
|
+
"Fix by de-duplicating before splitting: clean.dedup --method "
|
|
252
|
+
f"{method if method != 'exact' else 'phash'}, then dataset.split."
|
|
253
|
+
)
|
|
254
|
+
return result
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
HISTORY_PARAMS = (
|
|
258
|
+
Param("folder", "dir", "Dataset folder to read the history of.", required=True, must_exist=True),
|
|
259
|
+
Param("limit", "int", "How many entries to return, most recent last.", default=50, minimum=1),
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
@operation(
|
|
264
|
+
"inspect.history",
|
|
265
|
+
title="Dataset lineage",
|
|
266
|
+
category="inspect",
|
|
267
|
+
summary="Show what was done to this dataset, in order, with the settings each step used.",
|
|
268
|
+
params=HISTORY_PARAMS,
|
|
269
|
+
notes=(
|
|
270
|
+
"Every operation that writes a dataset appends to\n"
|
|
271
|
+
"<output>/.datafactory/history.jsonl, so a built folder carries its own\n"
|
|
272
|
+
"provenance. This reads it back.\n"
|
|
273
|
+
"\n"
|
|
274
|
+
"It answers the questions that are otherwise unanswerable six weeks\n"
|
|
275
|
+
"later: was this augmented, at what seed, was it resized before or\n"
|
|
276
|
+
"after the split, which folders went into the merge.\n"
|
|
277
|
+
"\n"
|
|
278
|
+
"The file is plain JSON Lines -- readable without this tool, and\n"
|
|
279
|
+
"appended to rather than rewritten, so a build that dies halfway still\n"
|
|
280
|
+
"leaves a truthful record of what it did manage."
|
|
281
|
+
),
|
|
282
|
+
)
|
|
283
|
+
def history(params: dict[str, Any], ctx: Context) -> OpResult:
|
|
284
|
+
result = OpResult(op="inspect.history", dry_run=ctx.dry_run)
|
|
285
|
+
folder: Path = params["folder"]
|
|
286
|
+
|
|
287
|
+
entries = provenance.read(folder)
|
|
288
|
+
if not entries:
|
|
289
|
+
result.warnings.append(
|
|
290
|
+
f"no history recorded under {folder}. Either it was not built by Data "
|
|
291
|
+
f"Factory, or it was built before lineage was recorded."
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
limited = entries[-params["limit"] :]
|
|
295
|
+
result.counts.update({"steps": len(entries), "shown": len(limited)})
|
|
296
|
+
result.data.update(
|
|
297
|
+
{
|
|
298
|
+
"history_file": provenance.history_path(folder),
|
|
299
|
+
"steps": limited,
|
|
300
|
+
"operations": [e.get("op", "?") for e in limited],
|
|
301
|
+
}
|
|
302
|
+
)
|
|
303
|
+
return result
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
# ------------------------------------------------------------------ helpers
|
|
307
|
+
|
|
308
|
+
def _split_groups(folder: Path, wanted: list[str] | None) -> dict[str, list[Path]]:
|
|
309
|
+
"""Images grouped by split folder.
|
|
310
|
+
|
|
311
|
+
Looks one and two levels down, so both ``train/`` and the YOLO
|
|
312
|
+
``images/train/`` layout are recognised without being told which it is.
|
|
313
|
+
"""
|
|
314
|
+
groups: dict[str, list[Path]] = {}
|
|
315
|
+
names = set(wanted or [])
|
|
316
|
+
|
|
317
|
+
candidates: list[Path] = [p for p in sorted(folder.iterdir()) if p.is_dir()]
|
|
318
|
+
images_dir = folder / "images"
|
|
319
|
+
if images_dir.is_dir():
|
|
320
|
+
nested = [p for p in sorted(images_dir.iterdir()) if p.is_dir()]
|
|
321
|
+
if nested:
|
|
322
|
+
candidates = nested
|
|
323
|
+
|
|
324
|
+
for directory in candidates:
|
|
325
|
+
if names and directory.name not in names:
|
|
326
|
+
continue
|
|
327
|
+
found = list(fs.iter_images(directory, recursive=True))
|
|
328
|
+
if found:
|
|
329
|
+
groups[directory.name] = found
|
|
330
|
+
return groups
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _fingerprinter(method: str, hash_size: int) -> Callable[[Path], Any]:
|
|
334
|
+
if method == "exact":
|
|
335
|
+
return fs.sha256_file
|
|
336
|
+
try:
|
|
337
|
+
import imagehash
|
|
338
|
+
from PIL import Image
|
|
339
|
+
except ImportError as exc: # pragma: no cover - depends on environment
|
|
340
|
+
raise RuntimeError(f"perceptual comparison needs Pillow and imagehash ({exc})") from exc
|
|
341
|
+
|
|
342
|
+
function = {
|
|
343
|
+
"phash": imagehash.phash,
|
|
344
|
+
"dhash": imagehash.dhash,
|
|
345
|
+
"ahash": imagehash.average_hash,
|
|
346
|
+
}[method]
|
|
347
|
+
|
|
348
|
+
def compute(path: Path) -> Any:
|
|
349
|
+
with Image.open(path) as image:
|
|
350
|
+
image.load()
|
|
351
|
+
return function(image, hash_size=hash_size)
|
|
352
|
+
|
|
353
|
+
return compute
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _exact_matches(
|
|
357
|
+
left: list[tuple[Any, Path]], right: list[tuple[Any, Path]]
|
|
358
|
+
) -> list[tuple[int, Path, Path]]:
|
|
359
|
+
index: dict[Any, Path] = {}
|
|
360
|
+
for digest, path in left:
|
|
361
|
+
index.setdefault(digest, path)
|
|
362
|
+
return [(0, index[digest], path) for digest, path in right if digest in index]
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _near_matches(
|
|
366
|
+
left: list[tuple[Any, Path]], right: list[tuple[Any, Path]], threshold: int
|
|
367
|
+
) -> list[tuple[int, Path, Path]]:
|
|
368
|
+
"""Radius search rather than all-pairs.
|
|
369
|
+
|
|
370
|
+
Comparing a 3,000-image train split against a 400-image val split pairwise
|
|
371
|
+
is 1.2M hash comparisons; the BK-tree turns each lookup into a handful.
|
|
372
|
+
"""
|
|
373
|
+
tree: BKTree = BKTree(lambda a, b: a[0] - b[0])
|
|
374
|
+
for entry in left:
|
|
375
|
+
tree.add(entry)
|
|
376
|
+
|
|
377
|
+
found: list[tuple[int, Path, Path]] = []
|
|
378
|
+
for digest, path in right:
|
|
379
|
+
nearest = tree.nearest((digest, path), threshold)
|
|
380
|
+
if nearest is not None:
|
|
381
|
+
distance, (_, other) = nearest
|
|
382
|
+
found.append((distance, other, path))
|
|
383
|
+
return found
|