easyannotate 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_factory/__init__.py +82 -0
- data_factory/__main__.py +6 -0
- data_factory/annotate/__init__.py +15 -0
- data_factory/annotate/attach.py +398 -0
- data_factory/annotate/demo.py +161 -0
- data_factory/annotate/images.py +443 -0
- data_factory/annotate/labels.py +399 -0
- data_factory/annotate/loop.py +359 -0
- data_factory/annotate/projects.py +207 -0
- data_factory/annotate/review.py +471 -0
- data_factory/annotate/schema.sql +171 -0
- data_factory/annotate/shapes.py +488 -0
- data_factory/annotate/store.py +292 -0
- data_factory/annotate/tools/__init__.py +39 -0
- data_factory/annotate/tools/fake.py +170 -0
- data_factory/annotate/tools/install.py +254 -0
- data_factory/annotate/tools/local.py +389 -0
- data_factory/annotate/tools/postprocess.py +175 -0
- data_factory/annotate/tools/protocol.py +238 -0
- data_factory/annotate/tools/registry.py +196 -0
- data_factory/annotate/tools/results.py +83 -0
- data_factory/annotate/tools/serve.py +106 -0
- data_factory/cli/__init__.py +21 -0
- data_factory/cli/main.py +837 -0
- data_factory/cli/menu.py +173 -0
- data_factory/cli/render.py +158 -0
- data_factory/core/__init__.py +11 -0
- data_factory/core/annotations/__init__.py +5 -0
- data_factory/core/annotations/coco.py +394 -0
- data_factory/core/annotations/csvfmt.py +174 -0
- data_factory/core/annotations/dota.py +98 -0
- data_factory/core/annotations/formats.py +626 -0
- data_factory/core/annotations/labelme.py +153 -0
- data_factory/core/annotations/model.py +334 -0
- data_factory/core/annotations/voc.py +231 -0
- data_factory/core/annotations/yolo.py +283 -0
- data_factory/core/bktree.py +89 -0
- data_factory/core/fs.py +245 -0
- data_factory/core/imaging.py +626 -0
- data_factory/core/ops/__init__.py +44 -0
- data_factory/core/ops/analyze.py +383 -0
- data_factory/core/ops/annotate_export.py +326 -0
- data_factory/core/ops/audit.py +274 -0
- data_factory/core/ops/augment.py +454 -0
- data_factory/core/ops/collect.py +241 -0
- data_factory/core/ops/combine.py +582 -0
- data_factory/core/ops/convert.py +212 -0
- data_factory/core/ops/dataset.py +246 -0
- data_factory/core/ops/dedup.py +235 -0
- data_factory/core/ops/export.py +168 -0
- data_factory/core/ops/image.py +407 -0
- data_factory/core/ops/labels.py +215 -0
- data_factory/core/ops/pairing.py +133 -0
- data_factory/core/ops/rename.py +154 -0
- data_factory/core/ops/report.py +323 -0
- data_factory/core/ops/visualize.py +164 -0
- data_factory/core/pairs.py +144 -0
- data_factory/core/parallel.py +191 -0
- data_factory/core/provenance.py +136 -0
- data_factory/core/registry.py +339 -0
- data_factory/core/stats.py +403 -0
- data_factory/core/types.py +148 -0
- data_factory/core/writer.py +194 -0
- data_factory/desktop.py +207 -0
- data_factory/hardware.py +243 -0
- data_factory/pipeline.py +192 -0
- data_factory/py.typed +0 -0
- data_factory/service/__init__.py +26 -0
- data_factory/service/app.py +483 -0
- data_factory/service/binding.py +174 -0
- data_factory/service/deps.py +133 -0
- data_factory/service/jobs.py +177 -0
- data_factory/service/limits.py +71 -0
- data_factory/service/routers/__init__.py +11 -0
- data_factory/service/routers/annotate.py +582 -0
- data_factory/service/routers/local.py +464 -0
- data_factory/service/routers/media.py +154 -0
- data_factory/service/routers/predictions.py +116 -0
- data_factory/service/routers/promote.py +182 -0
- data_factory/service/routers/tools.py +308 -0
- data_factory/service/static/annotate/api.js +246 -0
- data_factory/service/static/annotate/app.js +1696 -0
- data_factory/service/static/annotate/assist.js +106 -0
- data_factory/service/static/annotate/filmstrip.js +286 -0
- data_factory/service/static/annotate/hit.js +146 -0
- data_factory/service/static/annotate/labels.js +197 -0
- data_factory/service/static/annotate/render.js +286 -0
- data_factory/service/static/annotate/scene.js +277 -0
- data_factory/service/static/annotate/shapelist.js +72 -0
- data_factory/service/static/annotate/tools/box.js +96 -0
- data_factory/service/static/annotate/tools/line.js +96 -0
- data_factory/service/static/annotate/tools/obb.js +88 -0
- data_factory/service/static/annotate/tools/point.js +49 -0
- data_factory/service/static/annotate/tools/polygon.js +110 -0
- data_factory/service/static/annotate/tools/select.js +227 -0
- data_factory/service/static/annotate/undo.js +176 -0
- data_factory/service/static/annotate/viewport.js +156 -0
- data_factory/service/static/annotate.html +390 -0
- data_factory/service/static/index.html +632 -0
- data_factory/service/workspace.py +244 -0
- easyannotate/__init__.py +64 -0
- easyannotate/__main__.py +5 -0
- easyannotate/project.py +181 -0
- easyannotate/py.typed +0 -0
- easyannotate-0.4.0.dist-info/METADATA +767 -0
- easyannotate-0.4.0.dist-info/RECORD +109 -0
- easyannotate-0.4.0.dist-info/WHEEL +4 -0
- easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
- easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""Dataset format conversion -- any supported format to any other."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from ..annotations import formats
|
|
9
|
+
from ..annotations.model import AnnotationError
|
|
10
|
+
from ..registry import Param, operation
|
|
11
|
+
from ..types import Context, OpResult
|
|
12
|
+
|
|
13
|
+
FORMAT_CHOICES = ("auto",) + formats.FORMATS
|
|
14
|
+
|
|
15
|
+
CONVERT_PARAMS = (
|
|
16
|
+
Param("folder", "dir", "Dataset to convert.", required=True, must_exist=True),
|
|
17
|
+
Param("output", "dir", "Where the converted dataset is written.", required=True),
|
|
18
|
+
Param(
|
|
19
|
+
"to",
|
|
20
|
+
"choice",
|
|
21
|
+
"Target format.",
|
|
22
|
+
required=True,
|
|
23
|
+
choices=formats.FORMATS,
|
|
24
|
+
),
|
|
25
|
+
Param(
|
|
26
|
+
"source_format",
|
|
27
|
+
"choice",
|
|
28
|
+
"Source format; 'auto' sniffs the folder layout.",
|
|
29
|
+
default="auto",
|
|
30
|
+
choices=FORMAT_CHOICES,
|
|
31
|
+
),
|
|
32
|
+
Param(
|
|
33
|
+
"classes",
|
|
34
|
+
"strs",
|
|
35
|
+
"Explicit class order, e.g. 'Closed,Open'. Fixes YOLO indices in place.",
|
|
36
|
+
default=None,
|
|
37
|
+
),
|
|
38
|
+
Param("recursive", "bool", "Include sub-folders.", default=True),
|
|
39
|
+
Param("copy_images", "bool", "Copy the image files alongside the labels.", default=True),
|
|
40
|
+
Param(
|
|
41
|
+
"segmentation",
|
|
42
|
+
"bool",
|
|
43
|
+
"Keep polygon outlines where the target format supports them.",
|
|
44
|
+
default=True,
|
|
45
|
+
),
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@operation(
|
|
50
|
+
"convert.dataset",
|
|
51
|
+
title="Convert dataset format",
|
|
52
|
+
category="convert",
|
|
53
|
+
summary="Convert between Pascal VOC, YOLO, COCO, LabelMe and CSV in any direction.",
|
|
54
|
+
params=CONVERT_PARAMS,
|
|
55
|
+
destructive=True,
|
|
56
|
+
notes=(
|
|
57
|
+
"Formats meet in the middle at a neutral annotation model, so every\n"
|
|
58
|
+
"direction works: voc->yolo, coco->voc, labelme->coco, and so on.\n"
|
|
59
|
+
"\n"
|
|
60
|
+
"Things worth knowing before you run it:\n"
|
|
61
|
+
" * YOLO stores a class *index*, so the class list lives in data.yaml.\n"
|
|
62
|
+
" Reading a YOLO folder without one names classes class_0, class_1...\n"
|
|
63
|
+
" * COCO RLE / iscrowd masks cannot become polygons without a mask\n"
|
|
64
|
+
" decoder; those objects keep their boxes and the run warns.\n"
|
|
65
|
+
" * CSV cannot hold outlines, so converting a segmentation dataset to\n"
|
|
66
|
+
" csv is lossy -- it warns rather than dropping them silently.\n"
|
|
67
|
+
" * Splits are preserved when the source layout names them (a train/\n"
|
|
68
|
+
" sub-folder, or instances_val.json).\n"
|
|
69
|
+
"\n"
|
|
70
|
+
"About 'classes'. Without it, the YOLO class order is derived by sorting\n"
|
|
71
|
+
"the names found in *this* folder. That is reproducible but not stable:\n"
|
|
72
|
+
"add a class next month and every index at or after it shifts, so the\n"
|
|
73
|
+
".txt files you exported earlier quietly start meaning something else.\n"
|
|
74
|
+
"Pass the same 'classes' list on every export and the indices hold. Any\n"
|
|
75
|
+
"class found but not listed is appended at the end, never inserted."
|
|
76
|
+
),
|
|
77
|
+
)
|
|
78
|
+
def convert_dataset(params: dict[str, Any], ctx: Context) -> OpResult:
|
|
79
|
+
result = OpResult(op="convert.dataset", dry_run=ctx.dry_run)
|
|
80
|
+
folder: Path = params["folder"]
|
|
81
|
+
output: Path = params["output"]
|
|
82
|
+
target: str = params["to"]
|
|
83
|
+
|
|
84
|
+
try:
|
|
85
|
+
dataset = formats.read_dataset(
|
|
86
|
+
folder, params["source_format"], recursive=params["recursive"], ctx=ctx
|
|
87
|
+
)
|
|
88
|
+
except AnnotationError as exc:
|
|
89
|
+
result.fail(str(exc))
|
|
90
|
+
return result
|
|
91
|
+
|
|
92
|
+
result.warnings.extend(dataset.warnings)
|
|
93
|
+
if not dataset.items:
|
|
94
|
+
result.fail(f"no annotated images found in {folder}")
|
|
95
|
+
return result
|
|
96
|
+
|
|
97
|
+
if output.resolve() == folder.resolve():
|
|
98
|
+
result.fail("output must differ from folder -- converting in place would overwrite the source")
|
|
99
|
+
return result
|
|
100
|
+
|
|
101
|
+
# An explicit class order pins the YOLO indices. Applied after reading so
|
|
102
|
+
# it overrides a data.yaml too: the caller's declared order wins over
|
|
103
|
+
# whatever the source folder happened to carry.
|
|
104
|
+
declared: list[str] | None = params.get("classes")
|
|
105
|
+
if declared:
|
|
106
|
+
order = [name.strip() for name in declared if name.strip()]
|
|
107
|
+
found = {label for item in dataset.items for label in item.annotation.label_set}
|
|
108
|
+
unlisted = sorted(found - set(order))
|
|
109
|
+
if unlisted:
|
|
110
|
+
result.warnings.append(
|
|
111
|
+
f"{len(unlisted)} class(es) present but not in 'classes' were appended "
|
|
112
|
+
f"after the declared order: {', '.join(unlisted)}"
|
|
113
|
+
)
|
|
114
|
+
unused = [name for name in order if name not in found]
|
|
115
|
+
if unused:
|
|
116
|
+
# Keeping them is the point -- a class with no objects this month
|
|
117
|
+
# still has to hold its index for the files exported last month.
|
|
118
|
+
result.warnings.append(
|
|
119
|
+
f"{len(unused)} declared class(es) have no objects here but keep "
|
|
120
|
+
f"their index: {', '.join(unused)}"
|
|
121
|
+
)
|
|
122
|
+
dataset.classes = order + unlisted
|
|
123
|
+
|
|
124
|
+
stats = dataset.stats()
|
|
125
|
+
total = len(dataset.items)
|
|
126
|
+
ctx.emit(0, total, f"{dataset.source_format} -> {target}")
|
|
127
|
+
|
|
128
|
+
report = formats.write_dataset(
|
|
129
|
+
dataset,
|
|
130
|
+
output,
|
|
131
|
+
target,
|
|
132
|
+
ctx=ctx,
|
|
133
|
+
copy_images=params["copy_images"],
|
|
134
|
+
segmentation=params["segmentation"],
|
|
135
|
+
on_action=lambda action: result.actions.append(action),
|
|
136
|
+
)
|
|
137
|
+
ctx.emit(total, total, "written")
|
|
138
|
+
|
|
139
|
+
result.counts.update(
|
|
140
|
+
{
|
|
141
|
+
"images": stats["images"],
|
|
142
|
+
"objects": stats["objects"],
|
|
143
|
+
"classes": stats["classes"],
|
|
144
|
+
"backgrounds": stats["backgrounds"],
|
|
145
|
+
"labels_written": int(report["written"]),
|
|
146
|
+
"images_copied": int(report["images"]),
|
|
147
|
+
}
|
|
148
|
+
)
|
|
149
|
+
result.warnings.extend(str(w) for w in report["warnings"]) # type: ignore[union-attr]
|
|
150
|
+
if stats["missing_images"]:
|
|
151
|
+
result.warnings.append(
|
|
152
|
+
f"{stats['missing_images']} annotation(s) had no matching image file; "
|
|
153
|
+
f"their labels were still converted"
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
result.data.update(
|
|
157
|
+
{
|
|
158
|
+
"source_format": dataset.source_format,
|
|
159
|
+
"target_format": target,
|
|
160
|
+
"output": output,
|
|
161
|
+
"classes": dataset.labels,
|
|
162
|
+
}
|
|
163
|
+
)
|
|
164
|
+
return result
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
DETECT_PARAMS = (
|
|
168
|
+
Param("folder", "dir", "Dataset folder to identify.", required=True, must_exist=True),
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@operation(
|
|
173
|
+
"convert.detect",
|
|
174
|
+
title="Identify dataset format",
|
|
175
|
+
category="convert",
|
|
176
|
+
summary="Work out what annotation format a folder is in, and summarise it.",
|
|
177
|
+
params=DETECT_PARAMS,
|
|
178
|
+
notes=(
|
|
179
|
+
"Read-only. Useful before converting something you were handed: it\n"
|
|
180
|
+
"reports the detected format, the class list, and how many objects\n"
|
|
181
|
+
"carry polygons -- which is what decides whether a conversion is lossy."
|
|
182
|
+
),
|
|
183
|
+
)
|
|
184
|
+
def detect_format(params: dict[str, Any], ctx: Context) -> OpResult:
|
|
185
|
+
result = OpResult(op="convert.detect", dry_run=ctx.dry_run)
|
|
186
|
+
folder: Path = params["folder"]
|
|
187
|
+
|
|
188
|
+
try:
|
|
189
|
+
detected = formats.detect(folder)
|
|
190
|
+
except AnnotationError as exc:
|
|
191
|
+
result.fail(str(exc))
|
|
192
|
+
return result
|
|
193
|
+
|
|
194
|
+
try:
|
|
195
|
+
dataset = formats.read_dataset(folder, detected, recursive=True, ctx=ctx)
|
|
196
|
+
except AnnotationError as exc:
|
|
197
|
+
# Detection succeeded but the contents did not parse -- still useful.
|
|
198
|
+
result.data["format"] = detected
|
|
199
|
+
result.fail(str(exc))
|
|
200
|
+
return result
|
|
201
|
+
|
|
202
|
+
result.counts.update(dataset.stats())
|
|
203
|
+
result.warnings.extend(dataset.warnings)
|
|
204
|
+
result.data.update(
|
|
205
|
+
{
|
|
206
|
+
"format": detected,
|
|
207
|
+
"classes": dataset.labels,
|
|
208
|
+
"splits": sorted({i.split for i in dataset.items if i.split}),
|
|
209
|
+
"convertible_to": [f for f in formats.FORMATS if f != detected],
|
|
210
|
+
}
|
|
211
|
+
)
|
|
212
|
+
return result
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
"""Dataset arrangement: train/val/test splitting, and sorting by class.
|
|
2
|
+
|
|
3
|
+
The README listed train/test splitting as a supported capability but no script
|
|
4
|
+
ever implemented it. It is implemented here, and stratified by default -- an
|
|
5
|
+
unstratified random split of an imbalanced set (Project_Door has ~200 hard
|
|
6
|
+
negatives against ~3500 positives) can easily land almost every negative in one
|
|
7
|
+
side and skew the evaluation.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import random
|
|
13
|
+
from collections import defaultdict
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from .. import fs
|
|
18
|
+
from ..annotations import voc
|
|
19
|
+
from ..annotations.model import AnnotationError
|
|
20
|
+
from ..pairs import Pair, PairIndex
|
|
21
|
+
from ..registry import Param, operation
|
|
22
|
+
from ..types import Action, Context, OpResult
|
|
23
|
+
|
|
24
|
+
SPLIT_PARAMS = (
|
|
25
|
+
Param("folder", "dir", "Folder of image/label pairs to split.", required=True, must_exist=True),
|
|
26
|
+
Param("output", "dir", "Where the split folders are created.", required=True),
|
|
27
|
+
Param("ratios", "strs", "Split sizes, in order, e.g. 0.8,0.1,0.1", default="0.8,0.1,0.1"),
|
|
28
|
+
Param("names", "strs", "Names for each split.", default="train,val,test"),
|
|
29
|
+
Param("mode", "choice", "Copy into the split, or move.", default="copy", choices=("copy", "move")),
|
|
30
|
+
Param("seed", "int", "Random seed, so the same split is reproducible.", default=42),
|
|
31
|
+
Param("stratify", "bool", "Keep class proportions equal across splits.", default=True),
|
|
32
|
+
Param("layout", "choice", "Folder shape inside each split.",
|
|
33
|
+
default="flat", choices=("flat", "images_labels")),
|
|
34
|
+
Param("include", "choice", "Split complete pairs only, or all images.",
|
|
35
|
+
default="pairs", choices=("pairs", "all")),
|
|
36
|
+
Param("recursive", "bool", "Include sub-folders when scanning.", default=False),
|
|
37
|
+
Param("label_ext", "str", "Sidecar label extension.", default=".xml"),
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@operation(
|
|
42
|
+
"dataset.split",
|
|
43
|
+
title="Split into train / val / test",
|
|
44
|
+
category="dataset",
|
|
45
|
+
summary="Divide pairs into reproducible, class-balanced splits without separating a label from its image.",
|
|
46
|
+
params=SPLIT_PARAMS,
|
|
47
|
+
destructive=False,
|
|
48
|
+
notes=(
|
|
49
|
+
"Stratification groups by the set of classes in each image (an image with\n"
|
|
50
|
+
"both Open and Close is its own stratum, and background images are theirs),\n"
|
|
51
|
+
"then splits each group by the given ratios. Every split therefore sees the\n"
|
|
52
|
+
"same class mix as the whole set.\n\n"
|
|
53
|
+
"seed makes the split reproducible: same folder plus same seed gives the\n"
|
|
54
|
+
"same partition, which is what makes model comparisons meaningful."
|
|
55
|
+
),
|
|
56
|
+
)
|
|
57
|
+
def split(params: dict[str, Any], ctx: Context) -> OpResult:
|
|
58
|
+
result = OpResult(op="dataset.split", dry_run=ctx.dry_run)
|
|
59
|
+
folder: Path = params["folder"]
|
|
60
|
+
output: Path = params["output"]
|
|
61
|
+
|
|
62
|
+
try:
|
|
63
|
+
ratios = [float(r) for r in params["ratios"]]
|
|
64
|
+
except ValueError:
|
|
65
|
+
result.fail(f"ratios must be numbers, got {params['ratios']}")
|
|
66
|
+
return result
|
|
67
|
+
names = list(params["names"])
|
|
68
|
+
if len(ratios) != len(names):
|
|
69
|
+
result.fail(f"got {len(ratios)} ratios but {len(names)} names; they must match")
|
|
70
|
+
return result
|
|
71
|
+
if any(r < 0 for r in ratios) or sum(ratios) <= 0:
|
|
72
|
+
result.fail("ratios must be non-negative and sum to more than zero")
|
|
73
|
+
return result
|
|
74
|
+
ratios = [r / sum(ratios) for r in ratios]
|
|
75
|
+
|
|
76
|
+
index = PairIndex.scan(folder, recursive=params["recursive"], label_ext=params["label_ext"])
|
|
77
|
+
items = index.complete() if params["include"] == "pairs" else index.with_images()
|
|
78
|
+
result.counts["scanned"] = len(index)
|
|
79
|
+
result.counts["selected"] = len(items)
|
|
80
|
+
if not items:
|
|
81
|
+
result.warnings.append("nothing to split")
|
|
82
|
+
return result
|
|
83
|
+
|
|
84
|
+
strata = _stratify(items, params["stratify"], result)
|
|
85
|
+
rng = random.Random(int(params["seed"]))
|
|
86
|
+
buckets: dict[str, list[Pair]] = {name: [] for name in names}
|
|
87
|
+
|
|
88
|
+
for key in sorted(strata):
|
|
89
|
+
group = sorted(strata[key], key=lambda p: p.key)
|
|
90
|
+
rng.shuffle(group)
|
|
91
|
+
for name, chunk in zip(names, _partition(len(group), ratios)):
|
|
92
|
+
buckets[name].extend(group[chunk[0] : chunk[1]])
|
|
93
|
+
|
|
94
|
+
verb = Action.COPY if params["mode"] == "copy" else Action.MOVE
|
|
95
|
+
per_split_labels: dict[str, dict[str, int]] = {}
|
|
96
|
+
|
|
97
|
+
for name in names:
|
|
98
|
+
members = sorted(buckets[name], key=lambda p: p.key)
|
|
99
|
+
result.counts[name] = len(members)
|
|
100
|
+
label_counts: dict[str, int] = defaultdict(int)
|
|
101
|
+
|
|
102
|
+
# Create the folder even when the split is empty. Downstream steps
|
|
103
|
+
# address splits by name, and a missing folder turns an empty split into
|
|
104
|
+
# a hard failure three commands later.
|
|
105
|
+
if params["layout"] == "flat":
|
|
106
|
+
fs.ensure_dir(output / name, ctx)
|
|
107
|
+
else:
|
|
108
|
+
fs.ensure_dir(output / name / "images", ctx)
|
|
109
|
+
fs.ensure_dir(output / name / "labels", ctx)
|
|
110
|
+
|
|
111
|
+
if not members:
|
|
112
|
+
result.warnings.append(
|
|
113
|
+
f"split '{name}' is empty -- with {len(items)} item(s) across "
|
|
114
|
+
f"{len(strata)} stratum/strata, its share rounded to zero"
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
for position, pair in enumerate(members, start=1):
|
|
118
|
+
ctx.check_cancelled()
|
|
119
|
+
ctx.emit(position, len(members), f"{name}: {pair.stem}")
|
|
120
|
+
for path in pair.files:
|
|
121
|
+
destination = _destination(output, name, path, params["layout"], params["label_ext"])
|
|
122
|
+
fs.perform(result.record(verb, path, destination, f"split={name}"), ctx)
|
|
123
|
+
result.bump(f"{name}_files", len(pair.files))
|
|
124
|
+
for label in _labels_of(pair):
|
|
125
|
+
label_counts[label] += 1
|
|
126
|
+
|
|
127
|
+
per_split_labels[name] = dict(sorted(label_counts.items()))
|
|
128
|
+
|
|
129
|
+
result.data["per_split_labels"] = per_split_labels
|
|
130
|
+
result.data["ratios"] = dict(zip(names, ratios))
|
|
131
|
+
result.data["seed"] = int(params["seed"])
|
|
132
|
+
result.data["output"] = output
|
|
133
|
+
return result
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _destination(output: Path, split_name: str, path: Path, layout: str, label_ext: str) -> Path:
|
|
137
|
+
if layout == "flat":
|
|
138
|
+
return output / split_name / path.name
|
|
139
|
+
bucket = "labels" if path.suffix.lower() == label_ext.lower() else "images"
|
|
140
|
+
return output / split_name / bucket / path.name
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _labels_of(pair: Pair) -> list[str]:
|
|
144
|
+
if pair.label is None:
|
|
145
|
+
return []
|
|
146
|
+
try:
|
|
147
|
+
return sorted(voc.read(pair.label).label_set) or ["__background__"]
|
|
148
|
+
except AnnotationError:
|
|
149
|
+
return []
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _stratify(items: list[Pair], enabled: bool, result: OpResult) -> dict[str, list[Pair]]:
|
|
153
|
+
"""Group pairs by their class signature, or into one bucket if disabled."""
|
|
154
|
+
if not enabled:
|
|
155
|
+
return {"__all__": list(items)}
|
|
156
|
+
groups: dict[str, list[Pair]] = defaultdict(list)
|
|
157
|
+
for pair in items:
|
|
158
|
+
labels = _labels_of(pair)
|
|
159
|
+
groups["+".join(labels) if labels else "__unlabelled__"].append(pair)
|
|
160
|
+
result.data["strata"] = {k: len(v) for k, v in sorted(groups.items())}
|
|
161
|
+
return groups
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _partition(count: int, ratios: list[float]) -> list[tuple[int, int]]:
|
|
165
|
+
"""Slice boundaries for ``count`` items, with the remainder going first.
|
|
166
|
+
|
|
167
|
+
Largest-remainder allocation keeps tiny strata from vanishing: with 1 item
|
|
168
|
+
and 0.8/0.1/0.1, the item lands in train rather than being rounded away.
|
|
169
|
+
"""
|
|
170
|
+
exact = [count * r for r in ratios]
|
|
171
|
+
sizes = [int(value) for value in exact]
|
|
172
|
+
leftover = count - sum(sizes)
|
|
173
|
+
order = sorted(range(len(ratios)), key=lambda i: exact[i] - sizes[i], reverse=True)
|
|
174
|
+
for i in range(leftover):
|
|
175
|
+
sizes[order[i % len(order)]] += 1
|
|
176
|
+
|
|
177
|
+
bounds: list[tuple[int, int]] = []
|
|
178
|
+
cursor = 0
|
|
179
|
+
for size in sizes:
|
|
180
|
+
bounds.append((cursor, cursor + size))
|
|
181
|
+
cursor += size
|
|
182
|
+
return bounds
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
SORT_PARAMS = (
|
|
186
|
+
Param("folder", "dir", "Folder of image/label pairs.", required=True, must_exist=True),
|
|
187
|
+
Param("output", "dir", "Where the class folders are created.", required=True),
|
|
188
|
+
Param("mode", "choice", "Copy into class folders, or move.",
|
|
189
|
+
default="copy", choices=("copy", "move")),
|
|
190
|
+
Param("mixed_name", "str", "Folder name for images holding more than one class.",
|
|
191
|
+
default="mixed"),
|
|
192
|
+
Param("background_name", "str", "Folder name for images with no objects.",
|
|
193
|
+
default="background"),
|
|
194
|
+
Param("recursive", "bool", "Include sub-folders.", default=False),
|
|
195
|
+
Param("label_ext", "str", "Sidecar label extension.", default=".xml"),
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
@operation(
|
|
200
|
+
"dataset.sort",
|
|
201
|
+
title="Sort pairs into class folders",
|
|
202
|
+
category="dataset",
|
|
203
|
+
summary="Group image/label pairs into one folder per class, with mixed and background buckets.",
|
|
204
|
+
params=SORT_PARAMS,
|
|
205
|
+
notes=(
|
|
206
|
+
"A fast way to eyeball class balance and spot mislabelled images: browse\n"
|
|
207
|
+
"the Open folder and anything that is not an open door stands out."
|
|
208
|
+
),
|
|
209
|
+
)
|
|
210
|
+
def sort_by_class(params: dict[str, Any], ctx: Context) -> OpResult:
|
|
211
|
+
result = OpResult(op="dataset.sort", dry_run=ctx.dry_run)
|
|
212
|
+
folder: Path = params["folder"]
|
|
213
|
+
output: Path = params["output"]
|
|
214
|
+
verb = Action.COPY if params["mode"] == "copy" else Action.MOVE
|
|
215
|
+
|
|
216
|
+
index = PairIndex.scan(folder, recursive=params["recursive"], label_ext=params["label_ext"])
|
|
217
|
+
pairs = index.complete()
|
|
218
|
+
result.counts["scanned"] = len(index)
|
|
219
|
+
result.counts["sorted"] = 0
|
|
220
|
+
|
|
221
|
+
for position, pair in enumerate(pairs, start=1):
|
|
222
|
+
ctx.check_cancelled()
|
|
223
|
+
ctx.emit(position, len(pairs), pair.stem)
|
|
224
|
+
assert pair.label is not None
|
|
225
|
+
try:
|
|
226
|
+
annotation = voc.read(pair.label)
|
|
227
|
+
except AnnotationError as exc:
|
|
228
|
+
result.warnings.append(f"{pair.label.name}: {exc}")
|
|
229
|
+
result.bump("unreadable")
|
|
230
|
+
continue
|
|
231
|
+
|
|
232
|
+
labels = sorted(annotation.label_set)
|
|
233
|
+
if not labels:
|
|
234
|
+
bucket = params["background_name"]
|
|
235
|
+
elif len(labels) == 1:
|
|
236
|
+
bucket = labels[0]
|
|
237
|
+
else:
|
|
238
|
+
bucket = params["mixed_name"]
|
|
239
|
+
|
|
240
|
+
for path in pair.files:
|
|
241
|
+
fs.perform(result.record(verb, path, output / bucket / path.name, f"class={bucket}"), ctx)
|
|
242
|
+
result.bump("sorted")
|
|
243
|
+
result.bump(f"class:{bucket}")
|
|
244
|
+
|
|
245
|
+
result.data["output"] = output
|
|
246
|
+
return result
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
"""Duplicate removal -- exact and perceptual, single- or cross-folder.
|
|
2
|
+
|
|
3
|
+
This replaces two scripts that had diverged: one did cross-folder SHA-256 and
|
|
4
|
+
carried the sidecar XML along, the other did single-folder perceptual hashing
|
|
5
|
+
and had the XML-moving code commented out. Both behaviours are options here, so
|
|
6
|
+
a near-duplicate can never again be dropped while its annotation is left
|
|
7
|
+
behind.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from collections import defaultdict
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from .. import fs, parallel
|
|
18
|
+
from ..bktree import BKTree
|
|
19
|
+
from ..pairs import PairIndex, sidecar_for
|
|
20
|
+
from ..registry import Param, operation
|
|
21
|
+
from ..types import Action, Context, OpResult
|
|
22
|
+
|
|
23
|
+
#: Perceptual algorithms, resolved lazily so importing the package does not
|
|
24
|
+
#: require Pillow/imagehash unless a perceptual run is actually requested.
|
|
25
|
+
PERCEPTUAL = ("phash", "dhash", "ahash", "whash")
|
|
26
|
+
|
|
27
|
+
PARAMS = (
|
|
28
|
+
Param("folders", "dirs", "Folder(s) to scan. Several are compared against each other.",
|
|
29
|
+
required=True, must_exist=True),
|
|
30
|
+
Param("method", "choice", "How duplicates are identified.",
|
|
31
|
+
default="exact", choices=("exact",) + PERCEPTUAL),
|
|
32
|
+
Param("threshold", "int", "Max Hamming distance for perceptual matches (0 = identical).",
|
|
33
|
+
default=8, minimum=0, maximum=64),
|
|
34
|
+
Param("hash_size", "int", "Perceptual hash size; higher is stricter and slower.",
|
|
35
|
+
default=8, minimum=4, maximum=32),
|
|
36
|
+
Param("action", "choice", "What to do with the duplicates that are found.",
|
|
37
|
+
default="move", choices=("move", "delete", "report")),
|
|
38
|
+
Param("quarantine", "dir", "Where moved duplicates go. Defaults to <first folder>/_duplicates."),
|
|
39
|
+
Param("keep", "choice", "Which member of a duplicate group to keep.",
|
|
40
|
+
default="annotated", choices=("annotated", "first", "largest")),
|
|
41
|
+
Param("recursive", "bool", "Include sub-folders.", default=False),
|
|
42
|
+
Param("move_labels", "bool", "Move or delete the matching sidecar label too.", default=True),
|
|
43
|
+
Param("workers", "int", parallel.WORKERS_HELP, default=0, minimum=0, maximum=32),
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@operation(
|
|
48
|
+
"clean.dedup",
|
|
49
|
+
title="Remove duplicate images",
|
|
50
|
+
category="clean",
|
|
51
|
+
summary="Find exact or visually similar duplicates and quarantine them, sidecar labels included.",
|
|
52
|
+
params=PARAMS,
|
|
53
|
+
destructive=True,
|
|
54
|
+
safe_when=lambda p: p.get("action") == "report",
|
|
55
|
+
notes=(
|
|
56
|
+
"method=exact hashes file bytes: fast, catches only byte-identical copies.\n"
|
|
57
|
+
"method=phash and friends compare appearance, so re-compressed, resized or\n"
|
|
58
|
+
"lightly cropped copies are caught too -- tune with --threshold (0 is\n"
|
|
59
|
+
"identical, 8 is a good default, above ~12 starts merging genuinely\n"
|
|
60
|
+
"different images).\n\n"
|
|
61
|
+
"keep=annotated is the safe default: when a duplicate group contains both a\n"
|
|
62
|
+
"labelled and an unlabelled copy, the labelled one survives."
|
|
63
|
+
),
|
|
64
|
+
)
|
|
65
|
+
def dedup(params: dict[str, Any], ctx: Context) -> OpResult:
|
|
66
|
+
result = OpResult(op="clean.dedup", dry_run=ctx.dry_run)
|
|
67
|
+
folders: list[Path] = params["folders"]
|
|
68
|
+
method: str = params["method"]
|
|
69
|
+
recursive: bool = params["recursive"]
|
|
70
|
+
action: str = params["action"]
|
|
71
|
+
|
|
72
|
+
quarantine = params["quarantine"] or (folders[0] / "_duplicates")
|
|
73
|
+
|
|
74
|
+
# Gather every image across every folder, remembering which folder it came
|
|
75
|
+
# from so quarantined files can keep their origin in the path.
|
|
76
|
+
entries: list[tuple[Path, Path]] = []
|
|
77
|
+
for folder in folders:
|
|
78
|
+
index = PairIndex.scan(folder, recursive=recursive)
|
|
79
|
+
for pair in index.with_images():
|
|
80
|
+
assert pair.image is not None
|
|
81
|
+
entries.append((folder, pair.image))
|
|
82
|
+
|
|
83
|
+
total = len(entries)
|
|
84
|
+
# Seed the headline counters so they are present even at zero. A consumer
|
|
85
|
+
# reading result.as_dict() should never have to guess whether a missing key
|
|
86
|
+
# means "none found" or "this operation does not report that".
|
|
87
|
+
result.counts["scanned"] = total
|
|
88
|
+
result.counts["unique"] = 0
|
|
89
|
+
result.counts["duplicates"] = 0
|
|
90
|
+
if params["action"] == "delete":
|
|
91
|
+
result.counts["files_removed"] = 0
|
|
92
|
+
elif params["action"] == "move":
|
|
93
|
+
result.counts["files_moved"] = 0
|
|
94
|
+
if total == 0:
|
|
95
|
+
result.warnings.append("no images found in the supplied folder(s)")
|
|
96
|
+
return result
|
|
97
|
+
|
|
98
|
+
hasher = _exact_hasher() if method == "exact" else _perceptual_hasher(method, params["hash_size"])
|
|
99
|
+
threshold = 0 if method == "exact" else int(params["threshold"])
|
|
100
|
+
|
|
101
|
+
groups: dict[Any, list[tuple[Path, Path]]] = defaultdict(list)
|
|
102
|
+
tree: BKTree | None = None
|
|
103
|
+
representatives: dict[Any, Any] = {}
|
|
104
|
+
|
|
105
|
+
if method != "exact":
|
|
106
|
+
tree = BKTree(lambda a, b: a - b)
|
|
107
|
+
|
|
108
|
+
# Hashing is pure and decode-bound, so it runs across threads; grouping is
|
|
109
|
+
# order-dependent (the first image seen is the one kept) and stays serial.
|
|
110
|
+
# map_ordered yields in input order, so the two phases agree.
|
|
111
|
+
def note_failure(entry: tuple[Path, Path], exc: Exception) -> None:
|
|
112
|
+
result.warnings.append(f"skipped {entry[1].name}: {exc}")
|
|
113
|
+
result.bump("unreadable")
|
|
114
|
+
|
|
115
|
+
hashed = parallel.map_ordered(
|
|
116
|
+
entries,
|
|
117
|
+
lambda entry: hasher(entry[1]),
|
|
118
|
+
ctx=ctx,
|
|
119
|
+
workers=params["workers"],
|
|
120
|
+
label=lambda entry: f"hashing {entry[1].name}",
|
|
121
|
+
on_error=note_failure,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
for (folder, image), digest in hashed:
|
|
125
|
+
if digest is None:
|
|
126
|
+
continue
|
|
127
|
+
|
|
128
|
+
if method == "exact":
|
|
129
|
+
groups[digest].append((folder, image))
|
|
130
|
+
continue
|
|
131
|
+
|
|
132
|
+
assert tree is not None
|
|
133
|
+
match = tree.nearest(digest, threshold)
|
|
134
|
+
if match is None:
|
|
135
|
+
tree.add(digest)
|
|
136
|
+
representatives[_key(digest)] = digest
|
|
137
|
+
groups[_key(digest)].append((folder, image))
|
|
138
|
+
else:
|
|
139
|
+
groups[_key(match[1])].append((folder, image))
|
|
140
|
+
|
|
141
|
+
# Every group with more than one member yields duplicates.
|
|
142
|
+
duplicate_groups: dict[str, list[str]] = {}
|
|
143
|
+
taken: set[Path] = set()
|
|
144
|
+
|
|
145
|
+
for members in groups.values():
|
|
146
|
+
if len(members) < 2:
|
|
147
|
+
result.bump("unique")
|
|
148
|
+
continue
|
|
149
|
+
keeper, dropped = _choose_keeper(members, params["keep"], params["move_labels"])
|
|
150
|
+
result.bump("unique")
|
|
151
|
+
duplicate_groups[str(keeper[1])] = [str(p) for _, p in dropped]
|
|
152
|
+
|
|
153
|
+
for folder, image in dropped:
|
|
154
|
+
result.bump("duplicates")
|
|
155
|
+
targets = [image]
|
|
156
|
+
if params["move_labels"]:
|
|
157
|
+
label = sidecar_for(image)
|
|
158
|
+
if label.exists():
|
|
159
|
+
targets.append(label)
|
|
160
|
+
|
|
161
|
+
for target in targets:
|
|
162
|
+
if action == "report":
|
|
163
|
+
result.record(Action.SKIP, target, None, f"duplicate of {keeper[1].name}")
|
|
164
|
+
continue
|
|
165
|
+
if action == "delete":
|
|
166
|
+
file_action = result.record(
|
|
167
|
+
Action.DELETE, target, None, f"duplicate of {keeper[1].name}"
|
|
168
|
+
)
|
|
169
|
+
else:
|
|
170
|
+
relative = fs.relative_to_safe(target, folder)
|
|
171
|
+
destination = fs.unique_path(quarantine / folder.name / relative, taken=taken)
|
|
172
|
+
taken.add(destination)
|
|
173
|
+
file_action = result.record(
|
|
174
|
+
Action.MOVE, target, destination, f"duplicate of {keeper[1].name}"
|
|
175
|
+
)
|
|
176
|
+
fs.perform(file_action, ctx)
|
|
177
|
+
result.bump("files_removed" if action == "delete" else "files_moved")
|
|
178
|
+
|
|
179
|
+
result.data["groups"] = duplicate_groups
|
|
180
|
+
result.data["method"] = method
|
|
181
|
+
result.data["threshold"] = threshold
|
|
182
|
+
if action == "move":
|
|
183
|
+
result.data["quarantine"] = quarantine
|
|
184
|
+
return result
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _key(digest: Any) -> str:
|
|
188
|
+
return str(digest)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _choose_keeper(
|
|
192
|
+
members: list[tuple[Path, Path]], policy: str, consider_labels: bool
|
|
193
|
+
) -> tuple[tuple[Path, Path], list[tuple[Path, Path]]]:
|
|
194
|
+
"""Pick the survivor of a duplicate group; the rest are dropped."""
|
|
195
|
+
if policy == "annotated" and consider_labels:
|
|
196
|
+
# Prefer a copy that already has an annotation -- discarding the
|
|
197
|
+
# labelled twin and keeping a bare image throws away real work.
|
|
198
|
+
annotated = [m for m in members if sidecar_for(m[1]).exists()]
|
|
199
|
+
if annotated:
|
|
200
|
+
keeper = annotated[0]
|
|
201
|
+
return keeper, [m for m in members if m is not keeper]
|
|
202
|
+
if policy == "largest":
|
|
203
|
+
keeper = max(members, key=lambda m: m[1].stat().st_size)
|
|
204
|
+
return keeper, [m for m in members if m is not keeper]
|
|
205
|
+
keeper = members[0]
|
|
206
|
+
return keeper, list(members[1:])
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _exact_hasher() -> Callable[[Path], str]:
|
|
210
|
+
return fs.sha256_file
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _perceptual_hasher(method: str, hash_size: int) -> Callable[[Path], Any]:
|
|
214
|
+
try:
|
|
215
|
+
import imagehash
|
|
216
|
+
from PIL import Image
|
|
217
|
+
except ImportError as exc: # pragma: no cover - depends on environment
|
|
218
|
+
raise RuntimeError(
|
|
219
|
+
f"perceptual dedup needs Pillow and imagehash installed ({exc})"
|
|
220
|
+
) from exc
|
|
221
|
+
|
|
222
|
+
functions = {
|
|
223
|
+
"phash": imagehash.phash,
|
|
224
|
+
"dhash": imagehash.dhash,
|
|
225
|
+
"ahash": imagehash.average_hash,
|
|
226
|
+
"whash": imagehash.whash,
|
|
227
|
+
}
|
|
228
|
+
function = functions[method]
|
|
229
|
+
|
|
230
|
+
def compute(path: Path) -> Any:
|
|
231
|
+
with Image.open(path) as image:
|
|
232
|
+
image.load()
|
|
233
|
+
return function(image, hash_size=hash_size)
|
|
234
|
+
|
|
235
|
+
return compute
|