easyannotate 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_factory/__init__.py +82 -0
- data_factory/__main__.py +6 -0
- data_factory/annotate/__init__.py +15 -0
- data_factory/annotate/attach.py +398 -0
- data_factory/annotate/demo.py +161 -0
- data_factory/annotate/images.py +443 -0
- data_factory/annotate/labels.py +399 -0
- data_factory/annotate/loop.py +359 -0
- data_factory/annotate/projects.py +207 -0
- data_factory/annotate/review.py +471 -0
- data_factory/annotate/schema.sql +171 -0
- data_factory/annotate/shapes.py +488 -0
- data_factory/annotate/store.py +292 -0
- data_factory/annotate/tools/__init__.py +39 -0
- data_factory/annotate/tools/fake.py +170 -0
- data_factory/annotate/tools/install.py +254 -0
- data_factory/annotate/tools/local.py +389 -0
- data_factory/annotate/tools/postprocess.py +175 -0
- data_factory/annotate/tools/protocol.py +238 -0
- data_factory/annotate/tools/registry.py +196 -0
- data_factory/annotate/tools/results.py +83 -0
- data_factory/annotate/tools/serve.py +106 -0
- data_factory/cli/__init__.py +21 -0
- data_factory/cli/main.py +837 -0
- data_factory/cli/menu.py +173 -0
- data_factory/cli/render.py +158 -0
- data_factory/core/__init__.py +11 -0
- data_factory/core/annotations/__init__.py +5 -0
- data_factory/core/annotations/coco.py +394 -0
- data_factory/core/annotations/csvfmt.py +174 -0
- data_factory/core/annotations/dota.py +98 -0
- data_factory/core/annotations/formats.py +626 -0
- data_factory/core/annotations/labelme.py +153 -0
- data_factory/core/annotations/model.py +334 -0
- data_factory/core/annotations/voc.py +231 -0
- data_factory/core/annotations/yolo.py +283 -0
- data_factory/core/bktree.py +89 -0
- data_factory/core/fs.py +245 -0
- data_factory/core/imaging.py +626 -0
- data_factory/core/ops/__init__.py +44 -0
- data_factory/core/ops/analyze.py +383 -0
- data_factory/core/ops/annotate_export.py +326 -0
- data_factory/core/ops/audit.py +274 -0
- data_factory/core/ops/augment.py +454 -0
- data_factory/core/ops/collect.py +241 -0
- data_factory/core/ops/combine.py +582 -0
- data_factory/core/ops/convert.py +212 -0
- data_factory/core/ops/dataset.py +246 -0
- data_factory/core/ops/dedup.py +235 -0
- data_factory/core/ops/export.py +168 -0
- data_factory/core/ops/image.py +407 -0
- data_factory/core/ops/labels.py +215 -0
- data_factory/core/ops/pairing.py +133 -0
- data_factory/core/ops/rename.py +154 -0
- data_factory/core/ops/report.py +323 -0
- data_factory/core/ops/visualize.py +164 -0
- data_factory/core/pairs.py +144 -0
- data_factory/core/parallel.py +191 -0
- data_factory/core/provenance.py +136 -0
- data_factory/core/registry.py +339 -0
- data_factory/core/stats.py +403 -0
- data_factory/core/types.py +148 -0
- data_factory/core/writer.py +194 -0
- data_factory/desktop.py +207 -0
- data_factory/hardware.py +243 -0
- data_factory/pipeline.py +192 -0
- data_factory/py.typed +0 -0
- data_factory/service/__init__.py +26 -0
- data_factory/service/app.py +483 -0
- data_factory/service/binding.py +174 -0
- data_factory/service/deps.py +133 -0
- data_factory/service/jobs.py +177 -0
- data_factory/service/limits.py +71 -0
- data_factory/service/routers/__init__.py +11 -0
- data_factory/service/routers/annotate.py +582 -0
- data_factory/service/routers/local.py +464 -0
- data_factory/service/routers/media.py +154 -0
- data_factory/service/routers/predictions.py +116 -0
- data_factory/service/routers/promote.py +182 -0
- data_factory/service/routers/tools.py +308 -0
- data_factory/service/static/annotate/api.js +246 -0
- data_factory/service/static/annotate/app.js +1696 -0
- data_factory/service/static/annotate/assist.js +106 -0
- data_factory/service/static/annotate/filmstrip.js +286 -0
- data_factory/service/static/annotate/hit.js +146 -0
- data_factory/service/static/annotate/labels.js +197 -0
- data_factory/service/static/annotate/render.js +286 -0
- data_factory/service/static/annotate/scene.js +277 -0
- data_factory/service/static/annotate/shapelist.js +72 -0
- data_factory/service/static/annotate/tools/box.js +96 -0
- data_factory/service/static/annotate/tools/line.js +96 -0
- data_factory/service/static/annotate/tools/obb.js +88 -0
- data_factory/service/static/annotate/tools/point.js +49 -0
- data_factory/service/static/annotate/tools/polygon.js +110 -0
- data_factory/service/static/annotate/tools/select.js +227 -0
- data_factory/service/static/annotate/undo.js +176 -0
- data_factory/service/static/annotate/viewport.js +156 -0
- data_factory/service/static/annotate.html +390 -0
- data_factory/service/static/index.html +632 -0
- data_factory/service/workspace.py +244 -0
- easyannotate/__init__.py +64 -0
- easyannotate/__main__.py +5 -0
- easyannotate/project.py +181 -0
- easyannotate/py.typed +0 -0
- easyannotate-0.4.0.dist-info/METADATA +767 -0
- easyannotate-0.4.0.dist-info/RECORD +109 -0
- easyannotate-0.4.0.dist-info/WHEEL +4 -0
- easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
- easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,582 @@
|
|
|
1
|
+
"""Dataset algebra: merging, filtering and sampling.
|
|
2
|
+
|
|
3
|
+
These three cover most of what people actually do to a dataset between
|
|
4
|
+
collecting it and training on it, and all three are usually done with a
|
|
5
|
+
one-off script that gets the annotations subtly wrong -- merging two folders
|
|
6
|
+
whose files share names, or filtering by class and leaving the dropped class's
|
|
7
|
+
boxes behind in the label files.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import random
|
|
13
|
+
from collections import Counter
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from ..annotations import formats
|
|
18
|
+
from ..annotations.model import Annotation, AnnotationError
|
|
19
|
+
from ..registry import Param, operation
|
|
20
|
+
from ..types import Context, OpResult
|
|
21
|
+
from ..writer import DatasetWriter
|
|
22
|
+
from .image import OUTPUT_FORMATS, SOURCE_FORMATS
|
|
23
|
+
|
|
24
|
+
MERGE_PARAMS = (
|
|
25
|
+
Param("folders", "dirs", "Datasets to merge, in order.", required=True, must_exist=True),
|
|
26
|
+
Param("output", "dir", "Where the merged dataset is written.", required=True),
|
|
27
|
+
Param("source_format", "choice", "Input format for all sources.", default="auto", choices=SOURCE_FORMATS),
|
|
28
|
+
Param("format", "choice", "Output format.", default="same", choices=OUTPUT_FORMATS),
|
|
29
|
+
Param(
|
|
30
|
+
"label_map",
|
|
31
|
+
"mapping",
|
|
32
|
+
"Class renames applied during the merge, e.g. car=vehicle,truck=vehicle",
|
|
33
|
+
default=None,
|
|
34
|
+
),
|
|
35
|
+
Param("fold_case", "bool", "Treat classes differing only by case as one.", default=True),
|
|
36
|
+
Param(
|
|
37
|
+
"prefix_source",
|
|
38
|
+
"bool",
|
|
39
|
+
"Prefix filenames with their source folder name to guarantee uniqueness.",
|
|
40
|
+
default=True,
|
|
41
|
+
),
|
|
42
|
+
Param("keep_splits", "bool", "Preserve train/val/test folders from the sources.", default=True),
|
|
43
|
+
Param("recursive", "bool", "Include sub-folders.", default=True),
|
|
44
|
+
Param("quality", "int", "JPEG/WebP quality when re-encoding.", default=95, minimum=1, maximum=100),
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@operation(
|
|
49
|
+
"dataset.merge",
|
|
50
|
+
title="Merge datasets",
|
|
51
|
+
category="dataset",
|
|
52
|
+
summary="Combine several datasets into one, reconciling class names and filename collisions.",
|
|
53
|
+
params=MERGE_PARAMS,
|
|
54
|
+
destructive=True,
|
|
55
|
+
notes=(
|
|
56
|
+
"The two things that go wrong when merging by hand:\n"
|
|
57
|
+
"\n"
|
|
58
|
+
" Filename collisions. Two folders both containing image_001.jpg means\n"
|
|
59
|
+
" one silently overwrites the other, taking its annotations with it.\n"
|
|
60
|
+
" --prefix-source (on by default) makes every name unique.\n"
|
|
61
|
+
"\n"
|
|
62
|
+
" Class drift. The same object is 'car' in one set and 'Car' or\n"
|
|
63
|
+
" 'vehicle' in another, and you end up training three classes that\n"
|
|
64
|
+
" should be one. --fold-case handles capitalisation; --label-map\n"
|
|
65
|
+
" handles the rest: --label-map car=vehicle,truck=vehicle\n"
|
|
66
|
+
"\n"
|
|
67
|
+
"Sources may be in different formats -- each is detected separately."
|
|
68
|
+
),
|
|
69
|
+
)
|
|
70
|
+
def merge(params: dict[str, Any], ctx: Context) -> OpResult:
|
|
71
|
+
result = OpResult(op="dataset.merge", dry_run=ctx.dry_run)
|
|
72
|
+
folders: list[Path] = params["folders"]
|
|
73
|
+
output: Path = params["output"]
|
|
74
|
+
|
|
75
|
+
if len(folders) < 2:
|
|
76
|
+
result.fail("merging needs at least two source folders")
|
|
77
|
+
return result
|
|
78
|
+
resolved_out = output.resolve()
|
|
79
|
+
for folder in folders:
|
|
80
|
+
if folder.resolve() == resolved_out:
|
|
81
|
+
result.fail(f"output must differ from every source ({folder})")
|
|
82
|
+
return result
|
|
83
|
+
|
|
84
|
+
label_map: dict[str, str] = params["label_map"] or {}
|
|
85
|
+
fold_case: bool = params["fold_case"]
|
|
86
|
+
|
|
87
|
+
loaded: list[tuple[Path, formats.Dataset]] = []
|
|
88
|
+
for folder in folders:
|
|
89
|
+
try:
|
|
90
|
+
dataset = formats.read_dataset(
|
|
91
|
+
folder, params["source_format"], recursive=params["recursive"], ctx=ctx
|
|
92
|
+
)
|
|
93
|
+
except AnnotationError as exc:
|
|
94
|
+
result.fail(f"{folder}: {exc}")
|
|
95
|
+
return result
|
|
96
|
+
result.warnings.extend(f"{folder.name}: {w}" for w in dataset.warnings)
|
|
97
|
+
loaded.append((folder, dataset))
|
|
98
|
+
|
|
99
|
+
# Settle the class vocabulary across all sources before writing anything,
|
|
100
|
+
# so YOLO indices are stable and the mapping can be reported.
|
|
101
|
+
canonical: dict[str, str] = {}
|
|
102
|
+
for _, dataset in loaded:
|
|
103
|
+
for label in dataset.labels:
|
|
104
|
+
canonical.setdefault(label, _canonical(label, label_map, fold_case))
|
|
105
|
+
merged_classes = sorted(set(canonical.values()))
|
|
106
|
+
|
|
107
|
+
renames = {k: v for k, v in canonical.items() if k != v}
|
|
108
|
+
target_format = params["format"]
|
|
109
|
+
if target_format == "same":
|
|
110
|
+
target_format = loaded[0][1].source_format
|
|
111
|
+
|
|
112
|
+
writer = DatasetWriter(
|
|
113
|
+
output, target_format, ctx, classes=merged_classes, quality=params["quality"]
|
|
114
|
+
)
|
|
115
|
+
per_source: dict[str, int] = {}
|
|
116
|
+
seen_stems: set[str] = set()
|
|
117
|
+
collisions = 0
|
|
118
|
+
total = sum(len(d.items) for _, d in loaded)
|
|
119
|
+
position = 0
|
|
120
|
+
|
|
121
|
+
for folder, dataset in loaded:
|
|
122
|
+
written = 0
|
|
123
|
+
for item in dataset.items:
|
|
124
|
+
position += 1
|
|
125
|
+
ctx.check_cancelled()
|
|
126
|
+
ctx.emit(position, total, f"{folder.name}/{item.stem}")
|
|
127
|
+
|
|
128
|
+
stem = f"{folder.name}_{item.stem}" if params["prefix_source"] else item.stem
|
|
129
|
+
if stem in seen_stems:
|
|
130
|
+
collisions += 1
|
|
131
|
+
stem = f"{stem}_{collisions}"
|
|
132
|
+
seen_stems.add(stem)
|
|
133
|
+
|
|
134
|
+
writer.add(
|
|
135
|
+
_relabelled(item.annotation, canonical),
|
|
136
|
+
stem=stem,
|
|
137
|
+
source=item.image,
|
|
138
|
+
split=item.split if params["keep_splits"] else "",
|
|
139
|
+
suffix=item.image.suffix if item.image else ".jpg",
|
|
140
|
+
)
|
|
141
|
+
written += 1
|
|
142
|
+
per_source[str(folder)] = written
|
|
143
|
+
|
|
144
|
+
report = writer.close()
|
|
145
|
+
result.actions.extend(writer.actions)
|
|
146
|
+
result.counts.update(
|
|
147
|
+
{
|
|
148
|
+
"sources": len(loaded),
|
|
149
|
+
"images": total,
|
|
150
|
+
"written": report["labels"],
|
|
151
|
+
"classes": len(merged_classes),
|
|
152
|
+
"collisions_resolved": collisions,
|
|
153
|
+
}
|
|
154
|
+
)
|
|
155
|
+
result.warnings.extend(report["warnings"])
|
|
156
|
+
if collisions and not params["prefix_source"]:
|
|
157
|
+
result.warnings.append(
|
|
158
|
+
f"{collisions} filename collision(s) were renamed. Use --prefix-source to "
|
|
159
|
+
f"avoid them up front and keep names predictable."
|
|
160
|
+
)
|
|
161
|
+
result.data.update(
|
|
162
|
+
{
|
|
163
|
+
"output": output,
|
|
164
|
+
"format": target_format,
|
|
165
|
+
"classes": merged_classes,
|
|
166
|
+
"renamed_classes": renames,
|
|
167
|
+
"per_source": per_source,
|
|
168
|
+
}
|
|
169
|
+
)
|
|
170
|
+
if renames:
|
|
171
|
+
result.warnings.append(
|
|
172
|
+
"classes reconciled: "
|
|
173
|
+
+ ", ".join(f"{k} -> {v}" for k, v in sorted(renames.items())[:10])
|
|
174
|
+
)
|
|
175
|
+
return result
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _canonical(label: str, label_map: dict[str, str], fold_case: bool) -> str:
|
|
179
|
+
"""Map a source label onto its merged name."""
|
|
180
|
+
mapped = label_map.get(label)
|
|
181
|
+
if mapped is None and fold_case:
|
|
182
|
+
# Try a case-insensitive lookup, so --label-map Car=vehicle also
|
|
183
|
+
# catches "car" without listing every spelling.
|
|
184
|
+
for key, value in label_map.items():
|
|
185
|
+
if key.casefold() == label.casefold():
|
|
186
|
+
mapped = value
|
|
187
|
+
break
|
|
188
|
+
result = (mapped or label).strip()
|
|
189
|
+
return result
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _relabelled(annotation: Annotation, canonical: dict[str, str]) -> Annotation:
|
|
193
|
+
"""A copy whose shapes carry the merged class names.
|
|
194
|
+
|
|
195
|
+
Case folding is applied by canonicalising to whichever spelling sorts
|
|
196
|
+
first, which is reproducible and keeps the choice out of the file order.
|
|
197
|
+
"""
|
|
198
|
+
from ..annotations.model import Shape
|
|
199
|
+
|
|
200
|
+
shapes = [
|
|
201
|
+
Shape(
|
|
202
|
+
label=canonical.get(s.label, s.label),
|
|
203
|
+
bbox=s.bbox,
|
|
204
|
+
polygon=s.polygon,
|
|
205
|
+
pose=s.pose,
|
|
206
|
+
truncated=s.truncated,
|
|
207
|
+
difficult=s.difficult,
|
|
208
|
+
occluded=s.occluded,
|
|
209
|
+
)
|
|
210
|
+
for s in annotation.shapes
|
|
211
|
+
]
|
|
212
|
+
return Annotation(
|
|
213
|
+
filename=annotation.filename,
|
|
214
|
+
width=annotation.width,
|
|
215
|
+
height=annotation.height,
|
|
216
|
+
depth=annotation.depth,
|
|
217
|
+
shapes=shapes,
|
|
218
|
+
folder=annotation.folder,
|
|
219
|
+
source_path=annotation.source_path,
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
FILTER_PARAMS = (
|
|
224
|
+
Param("folder", "dir", "Dataset to filter.", required=True, must_exist=True),
|
|
225
|
+
Param("output", "dir", "Where the filtered dataset is written.", required=True),
|
|
226
|
+
Param("keep_classes", "strs", "Keep only these classes (blank means all).", default=None),
|
|
227
|
+
Param("drop_classes", "strs", "Remove these classes.", default=None),
|
|
228
|
+
Param("min_objects", "int", "Drop images with fewer objects than this.", default=0, minimum=0),
|
|
229
|
+
Param("max_objects", "int", "Drop images with more objects than this (0 = no limit).", default=0, minimum=0),
|
|
230
|
+
Param(
|
|
231
|
+
"min_area",
|
|
232
|
+
"int",
|
|
233
|
+
"Drop individual objects with a box area below this many pixels.",
|
|
234
|
+
default=0,
|
|
235
|
+
minimum=0,
|
|
236
|
+
),
|
|
237
|
+
Param("require_polygons", "bool", "Keep only objects that carry an outline.", default=False),
|
|
238
|
+
Param("keep_backgrounds", "bool", "Keep images that end up with no objects.", default=False),
|
|
239
|
+
Param("splits", "strs", "Keep only these splits (blank means all).", default=None),
|
|
240
|
+
Param("source_format", "choice", "Input format.", default="auto", choices=SOURCE_FORMATS),
|
|
241
|
+
Param("format", "choice", "Output format.", default="same", choices=OUTPUT_FORMATS),
|
|
242
|
+
Param("recursive", "bool", "Include sub-folders.", default=True),
|
|
243
|
+
Param("quality", "int", "JPEG/WebP quality.", default=95, minimum=1, maximum=100),
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
@operation(
|
|
248
|
+
"dataset.filter",
|
|
249
|
+
title="Filter dataset",
|
|
250
|
+
category="dataset",
|
|
251
|
+
summary="Build a subset by class, object count, object size or split.",
|
|
252
|
+
params=FILTER_PARAMS,
|
|
253
|
+
destructive=True,
|
|
254
|
+
notes=(
|
|
255
|
+
"Filtering happens at the object level first, then the image level. So\n"
|
|
256
|
+
"--keep-classes door removes every non-door box from the label files\n"
|
|
257
|
+
"*and then* drops any image left with nothing -- rather than leaving\n"
|
|
258
|
+
"images whose annotations still mention classes you excluded.\n"
|
|
259
|
+
"\n"
|
|
260
|
+
"--keep-backgrounds changes that last step: images emptied by the\n"
|
|
261
|
+
"filter are kept as hard negatives instead of dropped, which is often\n"
|
|
262
|
+
"what you want when narrowing a multi-class set to one class."
|
|
263
|
+
),
|
|
264
|
+
)
|
|
265
|
+
def filter_dataset(params: dict[str, Any], ctx: Context) -> OpResult:
|
|
266
|
+
result = OpResult(op="dataset.filter", dry_run=ctx.dry_run)
|
|
267
|
+
folder: Path = params["folder"]
|
|
268
|
+
output: Path = params["output"]
|
|
269
|
+
if output.resolve() == folder.resolve():
|
|
270
|
+
result.fail("output must differ from folder")
|
|
271
|
+
return result
|
|
272
|
+
|
|
273
|
+
try:
|
|
274
|
+
dataset = formats.read_dataset(
|
|
275
|
+
folder, params["source_format"], recursive=params["recursive"], ctx=ctx
|
|
276
|
+
)
|
|
277
|
+
except AnnotationError as exc:
|
|
278
|
+
result.fail(str(exc))
|
|
279
|
+
return result
|
|
280
|
+
result.warnings.extend(dataset.warnings)
|
|
281
|
+
|
|
282
|
+
keep = {c.casefold() for c in (params["keep_classes"] or [])}
|
|
283
|
+
drop = {c.casefold() for c in (params["drop_classes"] or [])}
|
|
284
|
+
splits = set(params["splits"] or [])
|
|
285
|
+
unknown = (keep | drop) - {label.casefold() for label in dataset.labels}
|
|
286
|
+
if unknown:
|
|
287
|
+
result.warnings.append(
|
|
288
|
+
f"class(es) not present in this dataset: {', '.join(sorted(unknown))}. "
|
|
289
|
+
f"Available: {', '.join(dataset.labels)}"
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
target_format = params["format"]
|
|
293
|
+
if target_format == "same":
|
|
294
|
+
target_format = dataset.source_format
|
|
295
|
+
|
|
296
|
+
kept_labels = [
|
|
297
|
+
label
|
|
298
|
+
for label in dataset.labels
|
|
299
|
+
if (not keep or label.casefold() in keep) and label.casefold() not in drop
|
|
300
|
+
]
|
|
301
|
+
writer = DatasetWriter(output, target_format, ctx, classes=kept_labels, quality=params["quality"])
|
|
302
|
+
|
|
303
|
+
total = len(dataset.items)
|
|
304
|
+
images_kept = 0
|
|
305
|
+
objects_kept = 0
|
|
306
|
+
objects_removed = 0
|
|
307
|
+
reasons: Counter[str] = Counter()
|
|
308
|
+
|
|
309
|
+
for position, item in enumerate(dataset.items, start=1):
|
|
310
|
+
ctx.check_cancelled()
|
|
311
|
+
ctx.emit(position, total, item.stem)
|
|
312
|
+
if splits and (item.split or "") not in splits:
|
|
313
|
+
reasons["split"] += 1
|
|
314
|
+
continue
|
|
315
|
+
|
|
316
|
+
shapes = []
|
|
317
|
+
for shape in item.annotation.shapes:
|
|
318
|
+
folded = shape.label.casefold()
|
|
319
|
+
if keep and folded not in keep:
|
|
320
|
+
objects_removed += 1
|
|
321
|
+
continue
|
|
322
|
+
if folded in drop:
|
|
323
|
+
objects_removed += 1
|
|
324
|
+
continue
|
|
325
|
+
if params["min_area"] and shape.bbox_area < params["min_area"]:
|
|
326
|
+
objects_removed += 1
|
|
327
|
+
continue
|
|
328
|
+
if params["require_polygons"] and not shape.has_polygon:
|
|
329
|
+
objects_removed += 1
|
|
330
|
+
continue
|
|
331
|
+
shapes.append(shape)
|
|
332
|
+
|
|
333
|
+
if len(shapes) < params["min_objects"]:
|
|
334
|
+
reasons["too few objects"] += 1
|
|
335
|
+
continue
|
|
336
|
+
if params["max_objects"] and len(shapes) > params["max_objects"]:
|
|
337
|
+
reasons["too many objects"] += 1
|
|
338
|
+
continue
|
|
339
|
+
if not shapes and not params["keep_backgrounds"]:
|
|
340
|
+
reasons["nothing left"] += 1
|
|
341
|
+
continue
|
|
342
|
+
|
|
343
|
+
writer.add(
|
|
344
|
+
Annotation(
|
|
345
|
+
filename=item.annotation.filename,
|
|
346
|
+
width=item.annotation.width,
|
|
347
|
+
height=item.annotation.height,
|
|
348
|
+
depth=item.annotation.depth,
|
|
349
|
+
shapes=shapes,
|
|
350
|
+
folder=item.annotation.folder,
|
|
351
|
+
),
|
|
352
|
+
stem=item.stem,
|
|
353
|
+
source=item.image,
|
|
354
|
+
split=item.split,
|
|
355
|
+
suffix=item.image.suffix if item.image else ".jpg",
|
|
356
|
+
)
|
|
357
|
+
images_kept += 1
|
|
358
|
+
objects_kept += len(shapes)
|
|
359
|
+
|
|
360
|
+
report = writer.close()
|
|
361
|
+
result.actions.extend(writer.actions)
|
|
362
|
+
result.counts.update(
|
|
363
|
+
{
|
|
364
|
+
"images_in": total,
|
|
365
|
+
"images_kept": images_kept,
|
|
366
|
+
"images_dropped": total - images_kept,
|
|
367
|
+
"objects_kept": objects_kept,
|
|
368
|
+
"objects_removed": objects_removed,
|
|
369
|
+
}
|
|
370
|
+
)
|
|
371
|
+
result.warnings.extend(report["warnings"])
|
|
372
|
+
result.data.update(
|
|
373
|
+
{
|
|
374
|
+
"output": output,
|
|
375
|
+
"format": target_format,
|
|
376
|
+
"classes": kept_labels,
|
|
377
|
+
"dropped_because": dict(reasons),
|
|
378
|
+
}
|
|
379
|
+
)
|
|
380
|
+
if images_kept == 0:
|
|
381
|
+
result.warnings.append("the filter kept nothing -- check the class names and thresholds")
|
|
382
|
+
return result
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
SAMPLE_PARAMS = (
|
|
386
|
+
Param("folder", "dir", "Dataset to sample from.", required=True, must_exist=True),
|
|
387
|
+
Param("output", "dir", "Where the sample is written.", required=True),
|
|
388
|
+
Param("count", "int", "How many images to take (0 = use fraction).", default=0, minimum=0),
|
|
389
|
+
Param(
|
|
390
|
+
"fraction",
|
|
391
|
+
"float",
|
|
392
|
+
"Fraction of the dataset to take, when count is 0.",
|
|
393
|
+
default=0.1,
|
|
394
|
+
minimum=0.0,
|
|
395
|
+
maximum=1.0,
|
|
396
|
+
),
|
|
397
|
+
Param(
|
|
398
|
+
"per_class",
|
|
399
|
+
"int",
|
|
400
|
+
"Cap on images per class -- the way to level an imbalanced set (0 = off).",
|
|
401
|
+
default=0,
|
|
402
|
+
minimum=0,
|
|
403
|
+
),
|
|
404
|
+
Param("stratify", "bool", "Preserve class proportions in the sample.", default=True),
|
|
405
|
+
Param("seed", "int", "Random seed; the same seed gives the same sample.", default=42),
|
|
406
|
+
Param("source_format", "choice", "Input format.", default="auto", choices=SOURCE_FORMATS),
|
|
407
|
+
Param("format", "choice", "Output format.", default="same", choices=OUTPUT_FORMATS),
|
|
408
|
+
Param("recursive", "bool", "Include sub-folders.", default=True),
|
|
409
|
+
Param("quality", "int", "JPEG/WebP quality.", default=95, minimum=1, maximum=100),
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
@operation(
|
|
414
|
+
"dataset.sample",
|
|
415
|
+
title="Sample a subset",
|
|
416
|
+
category="dataset",
|
|
417
|
+
summary="Take a reproducible subset -- random, stratified, or capped per class.",
|
|
418
|
+
params=SAMPLE_PARAMS,
|
|
419
|
+
destructive=True,
|
|
420
|
+
notes=(
|
|
421
|
+
"Three uses, in order of how often they come up:\n"
|
|
422
|
+
"\n"
|
|
423
|
+
" A small set to iterate on. --fraction 0.05 gives you something that\n"
|
|
424
|
+
" trains in minutes while you get the pipeline right.\n"
|
|
425
|
+
"\n"
|
|
426
|
+
" Levelling an imbalanced dataset. --per-class 500 caps the dominant\n"
|
|
427
|
+
" class without touching the rare ones, which is usually a better\n"
|
|
428
|
+
" starting point than class weights.\n"
|
|
429
|
+
"\n"
|
|
430
|
+
" A review sample. A stratified 200 images is enough to eyeball label\n"
|
|
431
|
+
" quality across every class.\n"
|
|
432
|
+
"\n"
|
|
433
|
+
"Always seeded, so 'the 500 images we evaluated on' stays meaningful."
|
|
434
|
+
),
|
|
435
|
+
)
|
|
436
|
+
def sample(params: dict[str, Any], ctx: Context) -> OpResult:
|
|
437
|
+
result = OpResult(op="dataset.sample", dry_run=ctx.dry_run)
|
|
438
|
+
folder: Path = params["folder"]
|
|
439
|
+
output: Path = params["output"]
|
|
440
|
+
if output.resolve() == folder.resolve():
|
|
441
|
+
result.fail("output must differ from folder")
|
|
442
|
+
return result
|
|
443
|
+
|
|
444
|
+
try:
|
|
445
|
+
dataset = formats.read_dataset(
|
|
446
|
+
folder, params["source_format"], recursive=params["recursive"], ctx=ctx
|
|
447
|
+
)
|
|
448
|
+
except AnnotationError as exc:
|
|
449
|
+
result.fail(str(exc))
|
|
450
|
+
return result
|
|
451
|
+
result.warnings.extend(dataset.warnings)
|
|
452
|
+
|
|
453
|
+
items = list(dataset.items)
|
|
454
|
+
if not items:
|
|
455
|
+
result.fail(f"no images found in {folder}")
|
|
456
|
+
return result
|
|
457
|
+
|
|
458
|
+
rng = random.Random(params["seed"])
|
|
459
|
+
if params["per_class"]:
|
|
460
|
+
chosen = _cap_per_class(items, params["per_class"], rng)
|
|
461
|
+
else:
|
|
462
|
+
wanted = params["count"] or max(1, round(len(items) * params["fraction"]))
|
|
463
|
+
wanted = min(wanted, len(items))
|
|
464
|
+
chosen = (
|
|
465
|
+
_stratified(items, wanted, rng) if params["stratify"] else rng.sample(items, wanted)
|
|
466
|
+
)
|
|
467
|
+
|
|
468
|
+
# Sorting restores a stable on-disk order after the random selection, so
|
|
469
|
+
# the output folder listing does not look shuffled.
|
|
470
|
+
chosen.sort(key=lambda i: i.stem)
|
|
471
|
+
|
|
472
|
+
target_format = params["format"]
|
|
473
|
+
if target_format == "same":
|
|
474
|
+
target_format = dataset.source_format
|
|
475
|
+
writer = DatasetWriter(
|
|
476
|
+
output, target_format, ctx, classes=dataset.labels, quality=params["quality"]
|
|
477
|
+
)
|
|
478
|
+
|
|
479
|
+
per_class: Counter[str] = Counter()
|
|
480
|
+
for position, item in enumerate(chosen, start=1):
|
|
481
|
+
ctx.check_cancelled()
|
|
482
|
+
ctx.emit(position, len(chosen), item.stem)
|
|
483
|
+
per_class.update(item.annotation.label_set)
|
|
484
|
+
writer.add(
|
|
485
|
+
item.annotation,
|
|
486
|
+
stem=item.stem,
|
|
487
|
+
source=item.image,
|
|
488
|
+
split=item.split,
|
|
489
|
+
suffix=item.image.suffix if item.image else ".jpg",
|
|
490
|
+
)
|
|
491
|
+
|
|
492
|
+
report = writer.close()
|
|
493
|
+
result.actions.extend(writer.actions)
|
|
494
|
+
result.counts.update(
|
|
495
|
+
{
|
|
496
|
+
"available": len(items),
|
|
497
|
+
"sampled": len(chosen),
|
|
498
|
+
"objects": sum(len(i.annotation.shapes) for i in chosen),
|
|
499
|
+
}
|
|
500
|
+
)
|
|
501
|
+
result.warnings.extend(report["warnings"])
|
|
502
|
+
result.data.update(
|
|
503
|
+
{
|
|
504
|
+
"output": output,
|
|
505
|
+
"format": target_format,
|
|
506
|
+
"seed": params["seed"],
|
|
507
|
+
"images_per_class": dict(sorted(per_class.items())),
|
|
508
|
+
}
|
|
509
|
+
)
|
|
510
|
+
return result
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def _stratified(items: list[formats.Item], wanted: int, rng: random.Random) -> list[formats.Item]:
|
|
514
|
+
"""Sample proportionally per class signature, so rare classes survive.
|
|
515
|
+
|
|
516
|
+
Grouping on the *set* of classes in an image, rather than on a single
|
|
517
|
+
class, is what keeps multi-label images from being counted twice.
|
|
518
|
+
"""
|
|
519
|
+
groups: dict[str, list[formats.Item]] = {}
|
|
520
|
+
for item in items:
|
|
521
|
+
key = ",".join(sorted(item.annotation.label_set)) or "(background)"
|
|
522
|
+
groups.setdefault(key, []).append(item)
|
|
523
|
+
|
|
524
|
+
chosen: list[formats.Item] = []
|
|
525
|
+
remainders: list[tuple[float, str]] = []
|
|
526
|
+
for key, members in sorted(groups.items()):
|
|
527
|
+
share = len(members) / len(items) * wanted
|
|
528
|
+
take = int(share)
|
|
529
|
+
rng.shuffle(members)
|
|
530
|
+
chosen.extend(members[:take])
|
|
531
|
+
remainders.append((share - take, key))
|
|
532
|
+
|
|
533
|
+
# Largest-remainder, so the total lands exactly on `wanted` rather than a
|
|
534
|
+
# few short from repeated rounding down.
|
|
535
|
+
remainders.sort(reverse=True)
|
|
536
|
+
index = 0
|
|
537
|
+
while len(chosen) < wanted and index < len(remainders):
|
|
538
|
+
key = remainders[index][1]
|
|
539
|
+
members = groups[key]
|
|
540
|
+
already = sum(1 for c in chosen if c in members)
|
|
541
|
+
if already < len(members):
|
|
542
|
+
chosen.append(members[already])
|
|
543
|
+
index += 1
|
|
544
|
+
return chosen[:wanted]
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def _cap_per_class(
|
|
548
|
+
items: list[formats.Item], cap: int, rng: random.Random
|
|
549
|
+
) -> list[formats.Item]:
|
|
550
|
+
"""Keep images until every class has reached ``cap``.
|
|
551
|
+
|
|
552
|
+
Rarest class first: an image showing both a common and a rare class should
|
|
553
|
+
be admitted on the rare class's account, or the rare class runs out of
|
|
554
|
+
candidates while the common one is still filling up.
|
|
555
|
+
"""
|
|
556
|
+
frequency: Counter[str] = Counter()
|
|
557
|
+
for item in items:
|
|
558
|
+
frequency.update(item.annotation.label_set)
|
|
559
|
+
|
|
560
|
+
order = sorted(
|
|
561
|
+
items,
|
|
562
|
+
key=lambda i: (
|
|
563
|
+
min((frequency[label] for label in i.annotation.label_set), default=10**9),
|
|
564
|
+
i.stem,
|
|
565
|
+
),
|
|
566
|
+
)
|
|
567
|
+
rng.shuffle(order)
|
|
568
|
+
order.sort(
|
|
569
|
+
key=lambda i: min((frequency[label] for label in i.annotation.label_set), default=10**9)
|
|
570
|
+
)
|
|
571
|
+
|
|
572
|
+
taken: Counter[str] = Counter()
|
|
573
|
+
chosen: list[formats.Item] = []
|
|
574
|
+
for item in order:
|
|
575
|
+
labels = item.annotation.label_set
|
|
576
|
+
if not labels:
|
|
577
|
+
continue
|
|
578
|
+
if all(taken[label] >= cap for label in labels):
|
|
579
|
+
continue
|
|
580
|
+
chosen.append(item)
|
|
581
|
+
taken.update(labels)
|
|
582
|
+
return chosen
|