easyannotate 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. data_factory/__init__.py +82 -0
  2. data_factory/__main__.py +6 -0
  3. data_factory/annotate/__init__.py +15 -0
  4. data_factory/annotate/attach.py +398 -0
  5. data_factory/annotate/demo.py +161 -0
  6. data_factory/annotate/images.py +443 -0
  7. data_factory/annotate/labels.py +399 -0
  8. data_factory/annotate/loop.py +359 -0
  9. data_factory/annotate/projects.py +207 -0
  10. data_factory/annotate/review.py +471 -0
  11. data_factory/annotate/schema.sql +171 -0
  12. data_factory/annotate/shapes.py +488 -0
  13. data_factory/annotate/store.py +292 -0
  14. data_factory/annotate/tools/__init__.py +39 -0
  15. data_factory/annotate/tools/fake.py +170 -0
  16. data_factory/annotate/tools/install.py +254 -0
  17. data_factory/annotate/tools/local.py +389 -0
  18. data_factory/annotate/tools/postprocess.py +175 -0
  19. data_factory/annotate/tools/protocol.py +238 -0
  20. data_factory/annotate/tools/registry.py +196 -0
  21. data_factory/annotate/tools/results.py +83 -0
  22. data_factory/annotate/tools/serve.py +106 -0
  23. data_factory/cli/__init__.py +21 -0
  24. data_factory/cli/main.py +837 -0
  25. data_factory/cli/menu.py +173 -0
  26. data_factory/cli/render.py +158 -0
  27. data_factory/core/__init__.py +11 -0
  28. data_factory/core/annotations/__init__.py +5 -0
  29. data_factory/core/annotations/coco.py +394 -0
  30. data_factory/core/annotations/csvfmt.py +174 -0
  31. data_factory/core/annotations/dota.py +98 -0
  32. data_factory/core/annotations/formats.py +626 -0
  33. data_factory/core/annotations/labelme.py +153 -0
  34. data_factory/core/annotations/model.py +334 -0
  35. data_factory/core/annotations/voc.py +231 -0
  36. data_factory/core/annotations/yolo.py +283 -0
  37. data_factory/core/bktree.py +89 -0
  38. data_factory/core/fs.py +245 -0
  39. data_factory/core/imaging.py +626 -0
  40. data_factory/core/ops/__init__.py +44 -0
  41. data_factory/core/ops/analyze.py +383 -0
  42. data_factory/core/ops/annotate_export.py +326 -0
  43. data_factory/core/ops/audit.py +274 -0
  44. data_factory/core/ops/augment.py +454 -0
  45. data_factory/core/ops/collect.py +241 -0
  46. data_factory/core/ops/combine.py +582 -0
  47. data_factory/core/ops/convert.py +212 -0
  48. data_factory/core/ops/dataset.py +246 -0
  49. data_factory/core/ops/dedup.py +235 -0
  50. data_factory/core/ops/export.py +168 -0
  51. data_factory/core/ops/image.py +407 -0
  52. data_factory/core/ops/labels.py +215 -0
  53. data_factory/core/ops/pairing.py +133 -0
  54. data_factory/core/ops/rename.py +154 -0
  55. data_factory/core/ops/report.py +323 -0
  56. data_factory/core/ops/visualize.py +164 -0
  57. data_factory/core/pairs.py +144 -0
  58. data_factory/core/parallel.py +191 -0
  59. data_factory/core/provenance.py +136 -0
  60. data_factory/core/registry.py +339 -0
  61. data_factory/core/stats.py +403 -0
  62. data_factory/core/types.py +148 -0
  63. data_factory/core/writer.py +194 -0
  64. data_factory/desktop.py +207 -0
  65. data_factory/hardware.py +243 -0
  66. data_factory/pipeline.py +192 -0
  67. data_factory/py.typed +0 -0
  68. data_factory/service/__init__.py +26 -0
  69. data_factory/service/app.py +483 -0
  70. data_factory/service/binding.py +174 -0
  71. data_factory/service/deps.py +133 -0
  72. data_factory/service/jobs.py +177 -0
  73. data_factory/service/limits.py +71 -0
  74. data_factory/service/routers/__init__.py +11 -0
  75. data_factory/service/routers/annotate.py +582 -0
  76. data_factory/service/routers/local.py +464 -0
  77. data_factory/service/routers/media.py +154 -0
  78. data_factory/service/routers/predictions.py +116 -0
  79. data_factory/service/routers/promote.py +182 -0
  80. data_factory/service/routers/tools.py +308 -0
  81. data_factory/service/static/annotate/api.js +246 -0
  82. data_factory/service/static/annotate/app.js +1696 -0
  83. data_factory/service/static/annotate/assist.js +106 -0
  84. data_factory/service/static/annotate/filmstrip.js +286 -0
  85. data_factory/service/static/annotate/hit.js +146 -0
  86. data_factory/service/static/annotate/labels.js +197 -0
  87. data_factory/service/static/annotate/render.js +286 -0
  88. data_factory/service/static/annotate/scene.js +277 -0
  89. data_factory/service/static/annotate/shapelist.js +72 -0
  90. data_factory/service/static/annotate/tools/box.js +96 -0
  91. data_factory/service/static/annotate/tools/line.js +96 -0
  92. data_factory/service/static/annotate/tools/obb.js +88 -0
  93. data_factory/service/static/annotate/tools/point.js +49 -0
  94. data_factory/service/static/annotate/tools/polygon.js +110 -0
  95. data_factory/service/static/annotate/tools/select.js +227 -0
  96. data_factory/service/static/annotate/undo.js +176 -0
  97. data_factory/service/static/annotate/viewport.js +156 -0
  98. data_factory/service/static/annotate.html +390 -0
  99. data_factory/service/static/index.html +632 -0
  100. data_factory/service/workspace.py +244 -0
  101. easyannotate/__init__.py +64 -0
  102. easyannotate/__main__.py +5 -0
  103. easyannotate/project.py +181 -0
  104. easyannotate/py.typed +0 -0
  105. easyannotate-0.4.0.dist-info/METADATA +767 -0
  106. easyannotate-0.4.0.dist-info/RECORD +109 -0
  107. easyannotate-0.4.0.dist-info/WHEEL +4 -0
  108. easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
  109. easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,212 @@
1
+ """Dataset format conversion -- any supported format to any other."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pathlib import Path
6
+ from typing import Any
7
+
8
+ from ..annotations import formats
9
+ from ..annotations.model import AnnotationError
10
+ from ..registry import Param, operation
11
+ from ..types import Context, OpResult
12
+
13
+ FORMAT_CHOICES = ("auto",) + formats.FORMATS
14
+
15
+ CONVERT_PARAMS = (
16
+ Param("folder", "dir", "Dataset to convert.", required=True, must_exist=True),
17
+ Param("output", "dir", "Where the converted dataset is written.", required=True),
18
+ Param(
19
+ "to",
20
+ "choice",
21
+ "Target format.",
22
+ required=True,
23
+ choices=formats.FORMATS,
24
+ ),
25
+ Param(
26
+ "source_format",
27
+ "choice",
28
+ "Source format; 'auto' sniffs the folder layout.",
29
+ default="auto",
30
+ choices=FORMAT_CHOICES,
31
+ ),
32
+ Param(
33
+ "classes",
34
+ "strs",
35
+ "Explicit class order, e.g. 'Closed,Open'. Fixes YOLO indices in place.",
36
+ default=None,
37
+ ),
38
+ Param("recursive", "bool", "Include sub-folders.", default=True),
39
+ Param("copy_images", "bool", "Copy the image files alongside the labels.", default=True),
40
+ Param(
41
+ "segmentation",
42
+ "bool",
43
+ "Keep polygon outlines where the target format supports them.",
44
+ default=True,
45
+ ),
46
+ )
47
+
48
+
49
+ @operation(
50
+ "convert.dataset",
51
+ title="Convert dataset format",
52
+ category="convert",
53
+ summary="Convert between Pascal VOC, YOLO, COCO, LabelMe and CSV in any direction.",
54
+ params=CONVERT_PARAMS,
55
+ destructive=True,
56
+ notes=(
57
+ "Formats meet in the middle at a neutral annotation model, so every\n"
58
+ "direction works: voc->yolo, coco->voc, labelme->coco, and so on.\n"
59
+ "\n"
60
+ "Things worth knowing before you run it:\n"
61
+ " * YOLO stores a class *index*, so the class list lives in data.yaml.\n"
62
+ " Reading a YOLO folder without one names classes class_0, class_1...\n"
63
+ " * COCO RLE / iscrowd masks cannot become polygons without a mask\n"
64
+ " decoder; those objects keep their boxes and the run warns.\n"
65
+ " * CSV cannot hold outlines, so converting a segmentation dataset to\n"
66
+ " csv is lossy -- it warns rather than dropping them silently.\n"
67
+ " * Splits are preserved when the source layout names them (a train/\n"
68
+ " sub-folder, or instances_val.json).\n"
69
+ "\n"
70
+ "About 'classes'. Without it, the YOLO class order is derived by sorting\n"
71
+ "the names found in *this* folder. That is reproducible but not stable:\n"
72
+ "add a class next month and every index at or after it shifts, so the\n"
73
+ ".txt files you exported earlier quietly start meaning something else.\n"
74
+ "Pass the same 'classes' list on every export and the indices hold. Any\n"
75
+ "class found but not listed is appended at the end, never inserted."
76
+ ),
77
+ )
78
+ def convert_dataset(params: dict[str, Any], ctx: Context) -> OpResult:
79
+ result = OpResult(op="convert.dataset", dry_run=ctx.dry_run)
80
+ folder: Path = params["folder"]
81
+ output: Path = params["output"]
82
+ target: str = params["to"]
83
+
84
+ try:
85
+ dataset = formats.read_dataset(
86
+ folder, params["source_format"], recursive=params["recursive"], ctx=ctx
87
+ )
88
+ except AnnotationError as exc:
89
+ result.fail(str(exc))
90
+ return result
91
+
92
+ result.warnings.extend(dataset.warnings)
93
+ if not dataset.items:
94
+ result.fail(f"no annotated images found in {folder}")
95
+ return result
96
+
97
+ if output.resolve() == folder.resolve():
98
+ result.fail("output must differ from folder -- converting in place would overwrite the source")
99
+ return result
100
+
101
+ # An explicit class order pins the YOLO indices. Applied after reading so
102
+ # it overrides a data.yaml too: the caller's declared order wins over
103
+ # whatever the source folder happened to carry.
104
+ declared: list[str] | None = params.get("classes")
105
+ if declared:
106
+ order = [name.strip() for name in declared if name.strip()]
107
+ found = {label for item in dataset.items for label in item.annotation.label_set}
108
+ unlisted = sorted(found - set(order))
109
+ if unlisted:
110
+ result.warnings.append(
111
+ f"{len(unlisted)} class(es) present but not in 'classes' were appended "
112
+ f"after the declared order: {', '.join(unlisted)}"
113
+ )
114
+ unused = [name for name in order if name not in found]
115
+ if unused:
116
+ # Keeping them is the point -- a class with no objects this month
117
+ # still has to hold its index for the files exported last month.
118
+ result.warnings.append(
119
+ f"{len(unused)} declared class(es) have no objects here but keep "
120
+ f"their index: {', '.join(unused)}"
121
+ )
122
+ dataset.classes = order + unlisted
123
+
124
+ stats = dataset.stats()
125
+ total = len(dataset.items)
126
+ ctx.emit(0, total, f"{dataset.source_format} -> {target}")
127
+
128
+ report = formats.write_dataset(
129
+ dataset,
130
+ output,
131
+ target,
132
+ ctx=ctx,
133
+ copy_images=params["copy_images"],
134
+ segmentation=params["segmentation"],
135
+ on_action=lambda action: result.actions.append(action),
136
+ )
137
+ ctx.emit(total, total, "written")
138
+
139
+ result.counts.update(
140
+ {
141
+ "images": stats["images"],
142
+ "objects": stats["objects"],
143
+ "classes": stats["classes"],
144
+ "backgrounds": stats["backgrounds"],
145
+ "labels_written": int(report["written"]),
146
+ "images_copied": int(report["images"]),
147
+ }
148
+ )
149
+ result.warnings.extend(str(w) for w in report["warnings"]) # type: ignore[union-attr]
150
+ if stats["missing_images"]:
151
+ result.warnings.append(
152
+ f"{stats['missing_images']} annotation(s) had no matching image file; "
153
+ f"their labels were still converted"
154
+ )
155
+
156
+ result.data.update(
157
+ {
158
+ "source_format": dataset.source_format,
159
+ "target_format": target,
160
+ "output": output,
161
+ "classes": dataset.labels,
162
+ }
163
+ )
164
+ return result
165
+
166
+
167
+ DETECT_PARAMS = (
168
+ Param("folder", "dir", "Dataset folder to identify.", required=True, must_exist=True),
169
+ )
170
+
171
+
172
+ @operation(
173
+ "convert.detect",
174
+ title="Identify dataset format",
175
+ category="convert",
176
+ summary="Work out what annotation format a folder is in, and summarise it.",
177
+ params=DETECT_PARAMS,
178
+ notes=(
179
+ "Read-only. Useful before converting something you were handed: it\n"
180
+ "reports the detected format, the class list, and how many objects\n"
181
+ "carry polygons -- which is what decides whether a conversion is lossy."
182
+ ),
183
+ )
184
+ def detect_format(params: dict[str, Any], ctx: Context) -> OpResult:
185
+ result = OpResult(op="convert.detect", dry_run=ctx.dry_run)
186
+ folder: Path = params["folder"]
187
+
188
+ try:
189
+ detected = formats.detect(folder)
190
+ except AnnotationError as exc:
191
+ result.fail(str(exc))
192
+ return result
193
+
194
+ try:
195
+ dataset = formats.read_dataset(folder, detected, recursive=True, ctx=ctx)
196
+ except AnnotationError as exc:
197
+ # Detection succeeded but the contents did not parse -- still useful.
198
+ result.data["format"] = detected
199
+ result.fail(str(exc))
200
+ return result
201
+
202
+ result.counts.update(dataset.stats())
203
+ result.warnings.extend(dataset.warnings)
204
+ result.data.update(
205
+ {
206
+ "format": detected,
207
+ "classes": dataset.labels,
208
+ "splits": sorted({i.split for i in dataset.items if i.split}),
209
+ "convertible_to": [f for f in formats.FORMATS if f != detected],
210
+ }
211
+ )
212
+ return result
@@ -0,0 +1,246 @@
1
+ """Dataset arrangement: train/val/test splitting, and sorting by class.
2
+
3
+ The README listed train/test splitting as a supported capability but no script
4
+ ever implemented it. It is implemented here, and stratified by default -- an
5
+ unstratified random split of an imbalanced set (Project_Door has ~200 hard
6
+ negatives against ~3500 positives) can easily land almost every negative in one
7
+ side and skew the evaluation.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import random
13
+ from collections import defaultdict
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ from .. import fs
18
+ from ..annotations import voc
19
+ from ..annotations.model import AnnotationError
20
+ from ..pairs import Pair, PairIndex
21
+ from ..registry import Param, operation
22
+ from ..types import Action, Context, OpResult
23
+
24
+ SPLIT_PARAMS = (
25
+ Param("folder", "dir", "Folder of image/label pairs to split.", required=True, must_exist=True),
26
+ Param("output", "dir", "Where the split folders are created.", required=True),
27
+ Param("ratios", "strs", "Split sizes, in order, e.g. 0.8,0.1,0.1", default="0.8,0.1,0.1"),
28
+ Param("names", "strs", "Names for each split.", default="train,val,test"),
29
+ Param("mode", "choice", "Copy into the split, or move.", default="copy", choices=("copy", "move")),
30
+ Param("seed", "int", "Random seed, so the same split is reproducible.", default=42),
31
+ Param("stratify", "bool", "Keep class proportions equal across splits.", default=True),
32
+ Param("layout", "choice", "Folder shape inside each split.",
33
+ default="flat", choices=("flat", "images_labels")),
34
+ Param("include", "choice", "Split complete pairs only, or all images.",
35
+ default="pairs", choices=("pairs", "all")),
36
+ Param("recursive", "bool", "Include sub-folders when scanning.", default=False),
37
+ Param("label_ext", "str", "Sidecar label extension.", default=".xml"),
38
+ )
39
+
40
+
41
+ @operation(
42
+ "dataset.split",
43
+ title="Split into train / val / test",
44
+ category="dataset",
45
+ summary="Divide pairs into reproducible, class-balanced splits without separating a label from its image.",
46
+ params=SPLIT_PARAMS,
47
+ destructive=False,
48
+ notes=(
49
+ "Stratification groups by the set of classes in each image (an image with\n"
50
+ "both Open and Close is its own stratum, and background images are theirs),\n"
51
+ "then splits each group by the given ratios. Every split therefore sees the\n"
52
+ "same class mix as the whole set.\n\n"
53
+ "seed makes the split reproducible: same folder plus same seed gives the\n"
54
+ "same partition, which is what makes model comparisons meaningful."
55
+ ),
56
+ )
57
+ def split(params: dict[str, Any], ctx: Context) -> OpResult:
58
+ result = OpResult(op="dataset.split", dry_run=ctx.dry_run)
59
+ folder: Path = params["folder"]
60
+ output: Path = params["output"]
61
+
62
+ try:
63
+ ratios = [float(r) for r in params["ratios"]]
64
+ except ValueError:
65
+ result.fail(f"ratios must be numbers, got {params['ratios']}")
66
+ return result
67
+ names = list(params["names"])
68
+ if len(ratios) != len(names):
69
+ result.fail(f"got {len(ratios)} ratios but {len(names)} names; they must match")
70
+ return result
71
+ if any(r < 0 for r in ratios) or sum(ratios) <= 0:
72
+ result.fail("ratios must be non-negative and sum to more than zero")
73
+ return result
74
+ ratios = [r / sum(ratios) for r in ratios]
75
+
76
+ index = PairIndex.scan(folder, recursive=params["recursive"], label_ext=params["label_ext"])
77
+ items = index.complete() if params["include"] == "pairs" else index.with_images()
78
+ result.counts["scanned"] = len(index)
79
+ result.counts["selected"] = len(items)
80
+ if not items:
81
+ result.warnings.append("nothing to split")
82
+ return result
83
+
84
+ strata = _stratify(items, params["stratify"], result)
85
+ rng = random.Random(int(params["seed"]))
86
+ buckets: dict[str, list[Pair]] = {name: [] for name in names}
87
+
88
+ for key in sorted(strata):
89
+ group = sorted(strata[key], key=lambda p: p.key)
90
+ rng.shuffle(group)
91
+ for name, chunk in zip(names, _partition(len(group), ratios)):
92
+ buckets[name].extend(group[chunk[0] : chunk[1]])
93
+
94
+ verb = Action.COPY if params["mode"] == "copy" else Action.MOVE
95
+ per_split_labels: dict[str, dict[str, int]] = {}
96
+
97
+ for name in names:
98
+ members = sorted(buckets[name], key=lambda p: p.key)
99
+ result.counts[name] = len(members)
100
+ label_counts: dict[str, int] = defaultdict(int)
101
+
102
+ # Create the folder even when the split is empty. Downstream steps
103
+ # address splits by name, and a missing folder turns an empty split into
104
+ # a hard failure three commands later.
105
+ if params["layout"] == "flat":
106
+ fs.ensure_dir(output / name, ctx)
107
+ else:
108
+ fs.ensure_dir(output / name / "images", ctx)
109
+ fs.ensure_dir(output / name / "labels", ctx)
110
+
111
+ if not members:
112
+ result.warnings.append(
113
+ f"split '{name}' is empty -- with {len(items)} item(s) across "
114
+ f"{len(strata)} stratum/strata, its share rounded to zero"
115
+ )
116
+
117
+ for position, pair in enumerate(members, start=1):
118
+ ctx.check_cancelled()
119
+ ctx.emit(position, len(members), f"{name}: {pair.stem}")
120
+ for path in pair.files:
121
+ destination = _destination(output, name, path, params["layout"], params["label_ext"])
122
+ fs.perform(result.record(verb, path, destination, f"split={name}"), ctx)
123
+ result.bump(f"{name}_files", len(pair.files))
124
+ for label in _labels_of(pair):
125
+ label_counts[label] += 1
126
+
127
+ per_split_labels[name] = dict(sorted(label_counts.items()))
128
+
129
+ result.data["per_split_labels"] = per_split_labels
130
+ result.data["ratios"] = dict(zip(names, ratios))
131
+ result.data["seed"] = int(params["seed"])
132
+ result.data["output"] = output
133
+ return result
134
+
135
+
136
+ def _destination(output: Path, split_name: str, path: Path, layout: str, label_ext: str) -> Path:
137
+ if layout == "flat":
138
+ return output / split_name / path.name
139
+ bucket = "labels" if path.suffix.lower() == label_ext.lower() else "images"
140
+ return output / split_name / bucket / path.name
141
+
142
+
143
+ def _labels_of(pair: Pair) -> list[str]:
144
+ if pair.label is None:
145
+ return []
146
+ try:
147
+ return sorted(voc.read(pair.label).label_set) or ["__background__"]
148
+ except AnnotationError:
149
+ return []
150
+
151
+
152
+ def _stratify(items: list[Pair], enabled: bool, result: OpResult) -> dict[str, list[Pair]]:
153
+ """Group pairs by their class signature, or into one bucket if disabled."""
154
+ if not enabled:
155
+ return {"__all__": list(items)}
156
+ groups: dict[str, list[Pair]] = defaultdict(list)
157
+ for pair in items:
158
+ labels = _labels_of(pair)
159
+ groups["+".join(labels) if labels else "__unlabelled__"].append(pair)
160
+ result.data["strata"] = {k: len(v) for k, v in sorted(groups.items())}
161
+ return groups
162
+
163
+
164
+ def _partition(count: int, ratios: list[float]) -> list[tuple[int, int]]:
165
+ """Slice boundaries for ``count`` items, with the remainder going first.
166
+
167
+ Largest-remainder allocation keeps tiny strata from vanishing: with 1 item
168
+ and 0.8/0.1/0.1, the item lands in train rather than being rounded away.
169
+ """
170
+ exact = [count * r for r in ratios]
171
+ sizes = [int(value) for value in exact]
172
+ leftover = count - sum(sizes)
173
+ order = sorted(range(len(ratios)), key=lambda i: exact[i] - sizes[i], reverse=True)
174
+ for i in range(leftover):
175
+ sizes[order[i % len(order)]] += 1
176
+
177
+ bounds: list[tuple[int, int]] = []
178
+ cursor = 0
179
+ for size in sizes:
180
+ bounds.append((cursor, cursor + size))
181
+ cursor += size
182
+ return bounds
183
+
184
+
185
+ SORT_PARAMS = (
186
+ Param("folder", "dir", "Folder of image/label pairs.", required=True, must_exist=True),
187
+ Param("output", "dir", "Where the class folders are created.", required=True),
188
+ Param("mode", "choice", "Copy into class folders, or move.",
189
+ default="copy", choices=("copy", "move")),
190
+ Param("mixed_name", "str", "Folder name for images holding more than one class.",
191
+ default="mixed"),
192
+ Param("background_name", "str", "Folder name for images with no objects.",
193
+ default="background"),
194
+ Param("recursive", "bool", "Include sub-folders.", default=False),
195
+ Param("label_ext", "str", "Sidecar label extension.", default=".xml"),
196
+ )
197
+
198
+
199
+ @operation(
200
+ "dataset.sort",
201
+ title="Sort pairs into class folders",
202
+ category="dataset",
203
+ summary="Group image/label pairs into one folder per class, with mixed and background buckets.",
204
+ params=SORT_PARAMS,
205
+ notes=(
206
+ "A fast way to eyeball class balance and spot mislabelled images: browse\n"
207
+ "the Open folder and anything that is not an open door stands out."
208
+ ),
209
+ )
210
+ def sort_by_class(params: dict[str, Any], ctx: Context) -> OpResult:
211
+ result = OpResult(op="dataset.sort", dry_run=ctx.dry_run)
212
+ folder: Path = params["folder"]
213
+ output: Path = params["output"]
214
+ verb = Action.COPY if params["mode"] == "copy" else Action.MOVE
215
+
216
+ index = PairIndex.scan(folder, recursive=params["recursive"], label_ext=params["label_ext"])
217
+ pairs = index.complete()
218
+ result.counts["scanned"] = len(index)
219
+ result.counts["sorted"] = 0
220
+
221
+ for position, pair in enumerate(pairs, start=1):
222
+ ctx.check_cancelled()
223
+ ctx.emit(position, len(pairs), pair.stem)
224
+ assert pair.label is not None
225
+ try:
226
+ annotation = voc.read(pair.label)
227
+ except AnnotationError as exc:
228
+ result.warnings.append(f"{pair.label.name}: {exc}")
229
+ result.bump("unreadable")
230
+ continue
231
+
232
+ labels = sorted(annotation.label_set)
233
+ if not labels:
234
+ bucket = params["background_name"]
235
+ elif len(labels) == 1:
236
+ bucket = labels[0]
237
+ else:
238
+ bucket = params["mixed_name"]
239
+
240
+ for path in pair.files:
241
+ fs.perform(result.record(verb, path, output / bucket / path.name, f"class={bucket}"), ctx)
242
+ result.bump("sorted")
243
+ result.bump(f"class:{bucket}")
244
+
245
+ result.data["output"] = output
246
+ return result
@@ -0,0 +1,235 @@
1
+ """Duplicate removal -- exact and perceptual, single- or cross-folder.
2
+
3
+ This replaces two scripts that had diverged: one did cross-folder SHA-256 and
4
+ carried the sidecar XML along, the other did single-folder perceptual hashing
5
+ and had the XML-moving code commented out. Both behaviours are options here, so
6
+ a near-duplicate can never again be dropped while its annotation is left
7
+ behind.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from collections import defaultdict
13
+ from collections.abc import Callable
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ from .. import fs, parallel
18
+ from ..bktree import BKTree
19
+ from ..pairs import PairIndex, sidecar_for
20
+ from ..registry import Param, operation
21
+ from ..types import Action, Context, OpResult
22
+
23
+ #: Perceptual algorithms, resolved lazily so importing the package does not
24
+ #: require Pillow/imagehash unless a perceptual run is actually requested.
25
+ PERCEPTUAL = ("phash", "dhash", "ahash", "whash")
26
+
27
+ PARAMS = (
28
+ Param("folders", "dirs", "Folder(s) to scan. Several are compared against each other.",
29
+ required=True, must_exist=True),
30
+ Param("method", "choice", "How duplicates are identified.",
31
+ default="exact", choices=("exact",) + PERCEPTUAL),
32
+ Param("threshold", "int", "Max Hamming distance for perceptual matches (0 = identical).",
33
+ default=8, minimum=0, maximum=64),
34
+ Param("hash_size", "int", "Perceptual hash size; higher is stricter and slower.",
35
+ default=8, minimum=4, maximum=32),
36
+ Param("action", "choice", "What to do with the duplicates that are found.",
37
+ default="move", choices=("move", "delete", "report")),
38
+ Param("quarantine", "dir", "Where moved duplicates go. Defaults to <first folder>/_duplicates."),
39
+ Param("keep", "choice", "Which member of a duplicate group to keep.",
40
+ default="annotated", choices=("annotated", "first", "largest")),
41
+ Param("recursive", "bool", "Include sub-folders.", default=False),
42
+ Param("move_labels", "bool", "Move or delete the matching sidecar label too.", default=True),
43
+ Param("workers", "int", parallel.WORKERS_HELP, default=0, minimum=0, maximum=32),
44
+ )
45
+
46
+
47
+ @operation(
48
+ "clean.dedup",
49
+ title="Remove duplicate images",
50
+ category="clean",
51
+ summary="Find exact or visually similar duplicates and quarantine them, sidecar labels included.",
52
+ params=PARAMS,
53
+ destructive=True,
54
+ safe_when=lambda p: p.get("action") == "report",
55
+ notes=(
56
+ "method=exact hashes file bytes: fast, catches only byte-identical copies.\n"
57
+ "method=phash and friends compare appearance, so re-compressed, resized or\n"
58
+ "lightly cropped copies are caught too -- tune with --threshold (0 is\n"
59
+ "identical, 8 is a good default, above ~12 starts merging genuinely\n"
60
+ "different images).\n\n"
61
+ "keep=annotated is the safe default: when a duplicate group contains both a\n"
62
+ "labelled and an unlabelled copy, the labelled one survives."
63
+ ),
64
+ )
65
+ def dedup(params: dict[str, Any], ctx: Context) -> OpResult:
66
+ result = OpResult(op="clean.dedup", dry_run=ctx.dry_run)
67
+ folders: list[Path] = params["folders"]
68
+ method: str = params["method"]
69
+ recursive: bool = params["recursive"]
70
+ action: str = params["action"]
71
+
72
+ quarantine = params["quarantine"] or (folders[0] / "_duplicates")
73
+
74
+ # Gather every image across every folder, remembering which folder it came
75
+ # from so quarantined files can keep their origin in the path.
76
+ entries: list[tuple[Path, Path]] = []
77
+ for folder in folders:
78
+ index = PairIndex.scan(folder, recursive=recursive)
79
+ for pair in index.with_images():
80
+ assert pair.image is not None
81
+ entries.append((folder, pair.image))
82
+
83
+ total = len(entries)
84
+ # Seed the headline counters so they are present even at zero. A consumer
85
+ # reading result.as_dict() should never have to guess whether a missing key
86
+ # means "none found" or "this operation does not report that".
87
+ result.counts["scanned"] = total
88
+ result.counts["unique"] = 0
89
+ result.counts["duplicates"] = 0
90
+ if params["action"] == "delete":
91
+ result.counts["files_removed"] = 0
92
+ elif params["action"] == "move":
93
+ result.counts["files_moved"] = 0
94
+ if total == 0:
95
+ result.warnings.append("no images found in the supplied folder(s)")
96
+ return result
97
+
98
+ hasher = _exact_hasher() if method == "exact" else _perceptual_hasher(method, params["hash_size"])
99
+ threshold = 0 if method == "exact" else int(params["threshold"])
100
+
101
+ groups: dict[Any, list[tuple[Path, Path]]] = defaultdict(list)
102
+ tree: BKTree | None = None
103
+ representatives: dict[Any, Any] = {}
104
+
105
+ if method != "exact":
106
+ tree = BKTree(lambda a, b: a - b)
107
+
108
+ # Hashing is pure and decode-bound, so it runs across threads; grouping is
109
+ # order-dependent (the first image seen is the one kept) and stays serial.
110
+ # map_ordered yields in input order, so the two phases agree.
111
+ def note_failure(entry: tuple[Path, Path], exc: Exception) -> None:
112
+ result.warnings.append(f"skipped {entry[1].name}: {exc}")
113
+ result.bump("unreadable")
114
+
115
+ hashed = parallel.map_ordered(
116
+ entries,
117
+ lambda entry: hasher(entry[1]),
118
+ ctx=ctx,
119
+ workers=params["workers"],
120
+ label=lambda entry: f"hashing {entry[1].name}",
121
+ on_error=note_failure,
122
+ )
123
+
124
+ for (folder, image), digest in hashed:
125
+ if digest is None:
126
+ continue
127
+
128
+ if method == "exact":
129
+ groups[digest].append((folder, image))
130
+ continue
131
+
132
+ assert tree is not None
133
+ match = tree.nearest(digest, threshold)
134
+ if match is None:
135
+ tree.add(digest)
136
+ representatives[_key(digest)] = digest
137
+ groups[_key(digest)].append((folder, image))
138
+ else:
139
+ groups[_key(match[1])].append((folder, image))
140
+
141
+ # Every group with more than one member yields duplicates.
142
+ duplicate_groups: dict[str, list[str]] = {}
143
+ taken: set[Path] = set()
144
+
145
+ for members in groups.values():
146
+ if len(members) < 2:
147
+ result.bump("unique")
148
+ continue
149
+ keeper, dropped = _choose_keeper(members, params["keep"], params["move_labels"])
150
+ result.bump("unique")
151
+ duplicate_groups[str(keeper[1])] = [str(p) for _, p in dropped]
152
+
153
+ for folder, image in dropped:
154
+ result.bump("duplicates")
155
+ targets = [image]
156
+ if params["move_labels"]:
157
+ label = sidecar_for(image)
158
+ if label.exists():
159
+ targets.append(label)
160
+
161
+ for target in targets:
162
+ if action == "report":
163
+ result.record(Action.SKIP, target, None, f"duplicate of {keeper[1].name}")
164
+ continue
165
+ if action == "delete":
166
+ file_action = result.record(
167
+ Action.DELETE, target, None, f"duplicate of {keeper[1].name}"
168
+ )
169
+ else:
170
+ relative = fs.relative_to_safe(target, folder)
171
+ destination = fs.unique_path(quarantine / folder.name / relative, taken=taken)
172
+ taken.add(destination)
173
+ file_action = result.record(
174
+ Action.MOVE, target, destination, f"duplicate of {keeper[1].name}"
175
+ )
176
+ fs.perform(file_action, ctx)
177
+ result.bump("files_removed" if action == "delete" else "files_moved")
178
+
179
+ result.data["groups"] = duplicate_groups
180
+ result.data["method"] = method
181
+ result.data["threshold"] = threshold
182
+ if action == "move":
183
+ result.data["quarantine"] = quarantine
184
+ return result
185
+
186
+
187
+ def _key(digest: Any) -> str:
188
+ return str(digest)
189
+
190
+
191
+ def _choose_keeper(
192
+ members: list[tuple[Path, Path]], policy: str, consider_labels: bool
193
+ ) -> tuple[tuple[Path, Path], list[tuple[Path, Path]]]:
194
+ """Pick the survivor of a duplicate group; the rest are dropped."""
195
+ if policy == "annotated" and consider_labels:
196
+ # Prefer a copy that already has an annotation -- discarding the
197
+ # labelled twin and keeping a bare image throws away real work.
198
+ annotated = [m for m in members if sidecar_for(m[1]).exists()]
199
+ if annotated:
200
+ keeper = annotated[0]
201
+ return keeper, [m for m in members if m is not keeper]
202
+ if policy == "largest":
203
+ keeper = max(members, key=lambda m: m[1].stat().st_size)
204
+ return keeper, [m for m in members if m is not keeper]
205
+ keeper = members[0]
206
+ return keeper, list(members[1:])
207
+
208
+
209
+ def _exact_hasher() -> Callable[[Path], str]:
210
+ return fs.sha256_file
211
+
212
+
213
+ def _perceptual_hasher(method: str, hash_size: int) -> Callable[[Path], Any]:
214
+ try:
215
+ import imagehash
216
+ from PIL import Image
217
+ except ImportError as exc: # pragma: no cover - depends on environment
218
+ raise RuntimeError(
219
+ f"perceptual dedup needs Pillow and imagehash installed ({exc})"
220
+ ) from exc
221
+
222
+ functions = {
223
+ "phash": imagehash.phash,
224
+ "dhash": imagehash.dhash,
225
+ "ahash": imagehash.average_hash,
226
+ "whash": imagehash.whash,
227
+ }
228
+ function = functions[method]
229
+
230
+ def compute(path: Path) -> Any:
231
+ with Image.open(path) as image:
232
+ image.load()
233
+ return function(image, hash_size=hash_size)
234
+
235
+ return compute