easyannotate 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. data_factory/__init__.py +82 -0
  2. data_factory/__main__.py +6 -0
  3. data_factory/annotate/__init__.py +15 -0
  4. data_factory/annotate/attach.py +398 -0
  5. data_factory/annotate/demo.py +161 -0
  6. data_factory/annotate/images.py +443 -0
  7. data_factory/annotate/labels.py +399 -0
  8. data_factory/annotate/loop.py +359 -0
  9. data_factory/annotate/projects.py +207 -0
  10. data_factory/annotate/review.py +471 -0
  11. data_factory/annotate/schema.sql +171 -0
  12. data_factory/annotate/shapes.py +488 -0
  13. data_factory/annotate/store.py +292 -0
  14. data_factory/annotate/tools/__init__.py +39 -0
  15. data_factory/annotate/tools/fake.py +170 -0
  16. data_factory/annotate/tools/install.py +254 -0
  17. data_factory/annotate/tools/local.py +389 -0
  18. data_factory/annotate/tools/postprocess.py +175 -0
  19. data_factory/annotate/tools/protocol.py +238 -0
  20. data_factory/annotate/tools/registry.py +196 -0
  21. data_factory/annotate/tools/results.py +83 -0
  22. data_factory/annotate/tools/serve.py +106 -0
  23. data_factory/cli/__init__.py +21 -0
  24. data_factory/cli/main.py +837 -0
  25. data_factory/cli/menu.py +173 -0
  26. data_factory/cli/render.py +158 -0
  27. data_factory/core/__init__.py +11 -0
  28. data_factory/core/annotations/__init__.py +5 -0
  29. data_factory/core/annotations/coco.py +394 -0
  30. data_factory/core/annotations/csvfmt.py +174 -0
  31. data_factory/core/annotations/dota.py +98 -0
  32. data_factory/core/annotations/formats.py +626 -0
  33. data_factory/core/annotations/labelme.py +153 -0
  34. data_factory/core/annotations/model.py +334 -0
  35. data_factory/core/annotations/voc.py +231 -0
  36. data_factory/core/annotations/yolo.py +283 -0
  37. data_factory/core/bktree.py +89 -0
  38. data_factory/core/fs.py +245 -0
  39. data_factory/core/imaging.py +626 -0
  40. data_factory/core/ops/__init__.py +44 -0
  41. data_factory/core/ops/analyze.py +383 -0
  42. data_factory/core/ops/annotate_export.py +326 -0
  43. data_factory/core/ops/audit.py +274 -0
  44. data_factory/core/ops/augment.py +454 -0
  45. data_factory/core/ops/collect.py +241 -0
  46. data_factory/core/ops/combine.py +582 -0
  47. data_factory/core/ops/convert.py +212 -0
  48. data_factory/core/ops/dataset.py +246 -0
  49. data_factory/core/ops/dedup.py +235 -0
  50. data_factory/core/ops/export.py +168 -0
  51. data_factory/core/ops/image.py +407 -0
  52. data_factory/core/ops/labels.py +215 -0
  53. data_factory/core/ops/pairing.py +133 -0
  54. data_factory/core/ops/rename.py +154 -0
  55. data_factory/core/ops/report.py +323 -0
  56. data_factory/core/ops/visualize.py +164 -0
  57. data_factory/core/pairs.py +144 -0
  58. data_factory/core/parallel.py +191 -0
  59. data_factory/core/provenance.py +136 -0
  60. data_factory/core/registry.py +339 -0
  61. data_factory/core/stats.py +403 -0
  62. data_factory/core/types.py +148 -0
  63. data_factory/core/writer.py +194 -0
  64. data_factory/desktop.py +207 -0
  65. data_factory/hardware.py +243 -0
  66. data_factory/pipeline.py +192 -0
  67. data_factory/py.typed +0 -0
  68. data_factory/service/__init__.py +26 -0
  69. data_factory/service/app.py +483 -0
  70. data_factory/service/binding.py +174 -0
  71. data_factory/service/deps.py +133 -0
  72. data_factory/service/jobs.py +177 -0
  73. data_factory/service/limits.py +71 -0
  74. data_factory/service/routers/__init__.py +11 -0
  75. data_factory/service/routers/annotate.py +582 -0
  76. data_factory/service/routers/local.py +464 -0
  77. data_factory/service/routers/media.py +154 -0
  78. data_factory/service/routers/predictions.py +116 -0
  79. data_factory/service/routers/promote.py +182 -0
  80. data_factory/service/routers/tools.py +308 -0
  81. data_factory/service/static/annotate/api.js +246 -0
  82. data_factory/service/static/annotate/app.js +1696 -0
  83. data_factory/service/static/annotate/assist.js +106 -0
  84. data_factory/service/static/annotate/filmstrip.js +286 -0
  85. data_factory/service/static/annotate/hit.js +146 -0
  86. data_factory/service/static/annotate/labels.js +197 -0
  87. data_factory/service/static/annotate/render.js +286 -0
  88. data_factory/service/static/annotate/scene.js +277 -0
  89. data_factory/service/static/annotate/shapelist.js +72 -0
  90. data_factory/service/static/annotate/tools/box.js +96 -0
  91. data_factory/service/static/annotate/tools/line.js +96 -0
  92. data_factory/service/static/annotate/tools/obb.js +88 -0
  93. data_factory/service/static/annotate/tools/point.js +49 -0
  94. data_factory/service/static/annotate/tools/polygon.js +110 -0
  95. data_factory/service/static/annotate/tools/select.js +227 -0
  96. data_factory/service/static/annotate/undo.js +176 -0
  97. data_factory/service/static/annotate/viewport.js +156 -0
  98. data_factory/service/static/annotate.html +390 -0
  99. data_factory/service/static/index.html +632 -0
  100. data_factory/service/workspace.py +244 -0
  101. easyannotate/__init__.py +64 -0
  102. easyannotate/__main__.py +5 -0
  103. easyannotate/project.py +181 -0
  104. easyannotate/py.typed +0 -0
  105. easyannotate-0.4.0.dist-info/METADATA +767 -0
  106. easyannotate-0.4.0.dist-info/RECORD +109 -0
  107. easyannotate-0.4.0.dist-info/WHEEL +4 -0
  108. easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
  109. easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,582 @@
1
+ """Dataset algebra: merging, filtering and sampling.
2
+
3
+ These three cover most of what people actually do to a dataset between
4
+ collecting it and training on it, and all three are usually done with a
5
+ one-off script that gets the annotations subtly wrong -- merging two folders
6
+ whose files share names, or filtering by class and leaving the dropped class's
7
+ boxes behind in the label files.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import random
13
+ from collections import Counter
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ from ..annotations import formats
18
+ from ..annotations.model import Annotation, AnnotationError
19
+ from ..registry import Param, operation
20
+ from ..types import Context, OpResult
21
+ from ..writer import DatasetWriter
22
+ from .image import OUTPUT_FORMATS, SOURCE_FORMATS
23
+
24
+ MERGE_PARAMS = (
25
+ Param("folders", "dirs", "Datasets to merge, in order.", required=True, must_exist=True),
26
+ Param("output", "dir", "Where the merged dataset is written.", required=True),
27
+ Param("source_format", "choice", "Input format for all sources.", default="auto", choices=SOURCE_FORMATS),
28
+ Param("format", "choice", "Output format.", default="same", choices=OUTPUT_FORMATS),
29
+ Param(
30
+ "label_map",
31
+ "mapping",
32
+ "Class renames applied during the merge, e.g. car=vehicle,truck=vehicle",
33
+ default=None,
34
+ ),
35
+ Param("fold_case", "bool", "Treat classes differing only by case as one.", default=True),
36
+ Param(
37
+ "prefix_source",
38
+ "bool",
39
+ "Prefix filenames with their source folder name to guarantee uniqueness.",
40
+ default=True,
41
+ ),
42
+ Param("keep_splits", "bool", "Preserve train/val/test folders from the sources.", default=True),
43
+ Param("recursive", "bool", "Include sub-folders.", default=True),
44
+ Param("quality", "int", "JPEG/WebP quality when re-encoding.", default=95, minimum=1, maximum=100),
45
+ )
46
+
47
+
48
+ @operation(
49
+ "dataset.merge",
50
+ title="Merge datasets",
51
+ category="dataset",
52
+ summary="Combine several datasets into one, reconciling class names and filename collisions.",
53
+ params=MERGE_PARAMS,
54
+ destructive=True,
55
+ notes=(
56
+ "The two things that go wrong when merging by hand:\n"
57
+ "\n"
58
+ " Filename collisions. Two folders both containing image_001.jpg means\n"
59
+ " one silently overwrites the other, taking its annotations with it.\n"
60
+ " --prefix-source (on by default) makes every name unique.\n"
61
+ "\n"
62
+ " Class drift. The same object is 'car' in one set and 'Car' or\n"
63
+ " 'vehicle' in another, and you end up training three classes that\n"
64
+ " should be one. --fold-case handles capitalisation; --label-map\n"
65
+ " handles the rest: --label-map car=vehicle,truck=vehicle\n"
66
+ "\n"
67
+ "Sources may be in different formats -- each is detected separately."
68
+ ),
69
+ )
70
+ def merge(params: dict[str, Any], ctx: Context) -> OpResult:
71
+ result = OpResult(op="dataset.merge", dry_run=ctx.dry_run)
72
+ folders: list[Path] = params["folders"]
73
+ output: Path = params["output"]
74
+
75
+ if len(folders) < 2:
76
+ result.fail("merging needs at least two source folders")
77
+ return result
78
+ resolved_out = output.resolve()
79
+ for folder in folders:
80
+ if folder.resolve() == resolved_out:
81
+ result.fail(f"output must differ from every source ({folder})")
82
+ return result
83
+
84
+ label_map: dict[str, str] = params["label_map"] or {}
85
+ fold_case: bool = params["fold_case"]
86
+
87
+ loaded: list[tuple[Path, formats.Dataset]] = []
88
+ for folder in folders:
89
+ try:
90
+ dataset = formats.read_dataset(
91
+ folder, params["source_format"], recursive=params["recursive"], ctx=ctx
92
+ )
93
+ except AnnotationError as exc:
94
+ result.fail(f"{folder}: {exc}")
95
+ return result
96
+ result.warnings.extend(f"{folder.name}: {w}" for w in dataset.warnings)
97
+ loaded.append((folder, dataset))
98
+
99
+ # Settle the class vocabulary across all sources before writing anything,
100
+ # so YOLO indices are stable and the mapping can be reported.
101
+ canonical: dict[str, str] = {}
102
+ for _, dataset in loaded:
103
+ for label in dataset.labels:
104
+ canonical.setdefault(label, _canonical(label, label_map, fold_case))
105
+ merged_classes = sorted(set(canonical.values()))
106
+
107
+ renames = {k: v for k, v in canonical.items() if k != v}
108
+ target_format = params["format"]
109
+ if target_format == "same":
110
+ target_format = loaded[0][1].source_format
111
+
112
+ writer = DatasetWriter(
113
+ output, target_format, ctx, classes=merged_classes, quality=params["quality"]
114
+ )
115
+ per_source: dict[str, int] = {}
116
+ seen_stems: set[str] = set()
117
+ collisions = 0
118
+ total = sum(len(d.items) for _, d in loaded)
119
+ position = 0
120
+
121
+ for folder, dataset in loaded:
122
+ written = 0
123
+ for item in dataset.items:
124
+ position += 1
125
+ ctx.check_cancelled()
126
+ ctx.emit(position, total, f"{folder.name}/{item.stem}")
127
+
128
+ stem = f"{folder.name}_{item.stem}" if params["prefix_source"] else item.stem
129
+ if stem in seen_stems:
130
+ collisions += 1
131
+ stem = f"{stem}_{collisions}"
132
+ seen_stems.add(stem)
133
+
134
+ writer.add(
135
+ _relabelled(item.annotation, canonical),
136
+ stem=stem,
137
+ source=item.image,
138
+ split=item.split if params["keep_splits"] else "",
139
+ suffix=item.image.suffix if item.image else ".jpg",
140
+ )
141
+ written += 1
142
+ per_source[str(folder)] = written
143
+
144
+ report = writer.close()
145
+ result.actions.extend(writer.actions)
146
+ result.counts.update(
147
+ {
148
+ "sources": len(loaded),
149
+ "images": total,
150
+ "written": report["labels"],
151
+ "classes": len(merged_classes),
152
+ "collisions_resolved": collisions,
153
+ }
154
+ )
155
+ result.warnings.extend(report["warnings"])
156
+ if collisions and not params["prefix_source"]:
157
+ result.warnings.append(
158
+ f"{collisions} filename collision(s) were renamed. Use --prefix-source to "
159
+ f"avoid them up front and keep names predictable."
160
+ )
161
+ result.data.update(
162
+ {
163
+ "output": output,
164
+ "format": target_format,
165
+ "classes": merged_classes,
166
+ "renamed_classes": renames,
167
+ "per_source": per_source,
168
+ }
169
+ )
170
+ if renames:
171
+ result.warnings.append(
172
+ "classes reconciled: "
173
+ + ", ".join(f"{k} -> {v}" for k, v in sorted(renames.items())[:10])
174
+ )
175
+ return result
176
+
177
+
178
+ def _canonical(label: str, label_map: dict[str, str], fold_case: bool) -> str:
179
+ """Map a source label onto its merged name."""
180
+ mapped = label_map.get(label)
181
+ if mapped is None and fold_case:
182
+ # Try a case-insensitive lookup, so --label-map Car=vehicle also
183
+ # catches "car" without listing every spelling.
184
+ for key, value in label_map.items():
185
+ if key.casefold() == label.casefold():
186
+ mapped = value
187
+ break
188
+ result = (mapped or label).strip()
189
+ return result
190
+
191
+
192
+ def _relabelled(annotation: Annotation, canonical: dict[str, str]) -> Annotation:
193
+ """A copy whose shapes carry the merged class names.
194
+
195
+ Case folding is applied by canonicalising to whichever spelling sorts
196
+ first, which is reproducible and keeps the choice out of the file order.
197
+ """
198
+ from ..annotations.model import Shape
199
+
200
+ shapes = [
201
+ Shape(
202
+ label=canonical.get(s.label, s.label),
203
+ bbox=s.bbox,
204
+ polygon=s.polygon,
205
+ pose=s.pose,
206
+ truncated=s.truncated,
207
+ difficult=s.difficult,
208
+ occluded=s.occluded,
209
+ )
210
+ for s in annotation.shapes
211
+ ]
212
+ return Annotation(
213
+ filename=annotation.filename,
214
+ width=annotation.width,
215
+ height=annotation.height,
216
+ depth=annotation.depth,
217
+ shapes=shapes,
218
+ folder=annotation.folder,
219
+ source_path=annotation.source_path,
220
+ )
221
+
222
+
223
+ FILTER_PARAMS = (
224
+ Param("folder", "dir", "Dataset to filter.", required=True, must_exist=True),
225
+ Param("output", "dir", "Where the filtered dataset is written.", required=True),
226
+ Param("keep_classes", "strs", "Keep only these classes (blank means all).", default=None),
227
+ Param("drop_classes", "strs", "Remove these classes.", default=None),
228
+ Param("min_objects", "int", "Drop images with fewer objects than this.", default=0, minimum=0),
229
+ Param("max_objects", "int", "Drop images with more objects than this (0 = no limit).", default=0, minimum=0),
230
+ Param(
231
+ "min_area",
232
+ "int",
233
+ "Drop individual objects with a box area below this many pixels.",
234
+ default=0,
235
+ minimum=0,
236
+ ),
237
+ Param("require_polygons", "bool", "Keep only objects that carry an outline.", default=False),
238
+ Param("keep_backgrounds", "bool", "Keep images that end up with no objects.", default=False),
239
+ Param("splits", "strs", "Keep only these splits (blank means all).", default=None),
240
+ Param("source_format", "choice", "Input format.", default="auto", choices=SOURCE_FORMATS),
241
+ Param("format", "choice", "Output format.", default="same", choices=OUTPUT_FORMATS),
242
+ Param("recursive", "bool", "Include sub-folders.", default=True),
243
+ Param("quality", "int", "JPEG/WebP quality.", default=95, minimum=1, maximum=100),
244
+ )
245
+
246
+
247
+ @operation(
248
+ "dataset.filter",
249
+ title="Filter dataset",
250
+ category="dataset",
251
+ summary="Build a subset by class, object count, object size or split.",
252
+ params=FILTER_PARAMS,
253
+ destructive=True,
254
+ notes=(
255
+ "Filtering happens at the object level first, then the image level. So\n"
256
+ "--keep-classes door removes every non-door box from the label files\n"
257
+ "*and then* drops any image left with nothing -- rather than leaving\n"
258
+ "images whose annotations still mention classes you excluded.\n"
259
+ "\n"
260
+ "--keep-backgrounds changes that last step: images emptied by the\n"
261
+ "filter are kept as hard negatives instead of dropped, which is often\n"
262
+ "what you want when narrowing a multi-class set to one class."
263
+ ),
264
+ )
265
+ def filter_dataset(params: dict[str, Any], ctx: Context) -> OpResult:
266
+ result = OpResult(op="dataset.filter", dry_run=ctx.dry_run)
267
+ folder: Path = params["folder"]
268
+ output: Path = params["output"]
269
+ if output.resolve() == folder.resolve():
270
+ result.fail("output must differ from folder")
271
+ return result
272
+
273
+ try:
274
+ dataset = formats.read_dataset(
275
+ folder, params["source_format"], recursive=params["recursive"], ctx=ctx
276
+ )
277
+ except AnnotationError as exc:
278
+ result.fail(str(exc))
279
+ return result
280
+ result.warnings.extend(dataset.warnings)
281
+
282
+ keep = {c.casefold() for c in (params["keep_classes"] or [])}
283
+ drop = {c.casefold() for c in (params["drop_classes"] or [])}
284
+ splits = set(params["splits"] or [])
285
+ unknown = (keep | drop) - {label.casefold() for label in dataset.labels}
286
+ if unknown:
287
+ result.warnings.append(
288
+ f"class(es) not present in this dataset: {', '.join(sorted(unknown))}. "
289
+ f"Available: {', '.join(dataset.labels)}"
290
+ )
291
+
292
+ target_format = params["format"]
293
+ if target_format == "same":
294
+ target_format = dataset.source_format
295
+
296
+ kept_labels = [
297
+ label
298
+ for label in dataset.labels
299
+ if (not keep or label.casefold() in keep) and label.casefold() not in drop
300
+ ]
301
+ writer = DatasetWriter(output, target_format, ctx, classes=kept_labels, quality=params["quality"])
302
+
303
+ total = len(dataset.items)
304
+ images_kept = 0
305
+ objects_kept = 0
306
+ objects_removed = 0
307
+ reasons: Counter[str] = Counter()
308
+
309
+ for position, item in enumerate(dataset.items, start=1):
310
+ ctx.check_cancelled()
311
+ ctx.emit(position, total, item.stem)
312
+ if splits and (item.split or "") not in splits:
313
+ reasons["split"] += 1
314
+ continue
315
+
316
+ shapes = []
317
+ for shape in item.annotation.shapes:
318
+ folded = shape.label.casefold()
319
+ if keep and folded not in keep:
320
+ objects_removed += 1
321
+ continue
322
+ if folded in drop:
323
+ objects_removed += 1
324
+ continue
325
+ if params["min_area"] and shape.bbox_area < params["min_area"]:
326
+ objects_removed += 1
327
+ continue
328
+ if params["require_polygons"] and not shape.has_polygon:
329
+ objects_removed += 1
330
+ continue
331
+ shapes.append(shape)
332
+
333
+ if len(shapes) < params["min_objects"]:
334
+ reasons["too few objects"] += 1
335
+ continue
336
+ if params["max_objects"] and len(shapes) > params["max_objects"]:
337
+ reasons["too many objects"] += 1
338
+ continue
339
+ if not shapes and not params["keep_backgrounds"]:
340
+ reasons["nothing left"] += 1
341
+ continue
342
+
343
+ writer.add(
344
+ Annotation(
345
+ filename=item.annotation.filename,
346
+ width=item.annotation.width,
347
+ height=item.annotation.height,
348
+ depth=item.annotation.depth,
349
+ shapes=shapes,
350
+ folder=item.annotation.folder,
351
+ ),
352
+ stem=item.stem,
353
+ source=item.image,
354
+ split=item.split,
355
+ suffix=item.image.suffix if item.image else ".jpg",
356
+ )
357
+ images_kept += 1
358
+ objects_kept += len(shapes)
359
+
360
+ report = writer.close()
361
+ result.actions.extend(writer.actions)
362
+ result.counts.update(
363
+ {
364
+ "images_in": total,
365
+ "images_kept": images_kept,
366
+ "images_dropped": total - images_kept,
367
+ "objects_kept": objects_kept,
368
+ "objects_removed": objects_removed,
369
+ }
370
+ )
371
+ result.warnings.extend(report["warnings"])
372
+ result.data.update(
373
+ {
374
+ "output": output,
375
+ "format": target_format,
376
+ "classes": kept_labels,
377
+ "dropped_because": dict(reasons),
378
+ }
379
+ )
380
+ if images_kept == 0:
381
+ result.warnings.append("the filter kept nothing -- check the class names and thresholds")
382
+ return result
383
+
384
+
385
+ SAMPLE_PARAMS = (
386
+ Param("folder", "dir", "Dataset to sample from.", required=True, must_exist=True),
387
+ Param("output", "dir", "Where the sample is written.", required=True),
388
+ Param("count", "int", "How many images to take (0 = use fraction).", default=0, minimum=0),
389
+ Param(
390
+ "fraction",
391
+ "float",
392
+ "Fraction of the dataset to take, when count is 0.",
393
+ default=0.1,
394
+ minimum=0.0,
395
+ maximum=1.0,
396
+ ),
397
+ Param(
398
+ "per_class",
399
+ "int",
400
+ "Cap on images per class -- the way to level an imbalanced set (0 = off).",
401
+ default=0,
402
+ minimum=0,
403
+ ),
404
+ Param("stratify", "bool", "Preserve class proportions in the sample.", default=True),
405
+ Param("seed", "int", "Random seed; the same seed gives the same sample.", default=42),
406
+ Param("source_format", "choice", "Input format.", default="auto", choices=SOURCE_FORMATS),
407
+ Param("format", "choice", "Output format.", default="same", choices=OUTPUT_FORMATS),
408
+ Param("recursive", "bool", "Include sub-folders.", default=True),
409
+ Param("quality", "int", "JPEG/WebP quality.", default=95, minimum=1, maximum=100),
410
+ )
411
+
412
+
413
+ @operation(
414
+ "dataset.sample",
415
+ title="Sample a subset",
416
+ category="dataset",
417
+ summary="Take a reproducible subset -- random, stratified, or capped per class.",
418
+ params=SAMPLE_PARAMS,
419
+ destructive=True,
420
+ notes=(
421
+ "Three uses, in order of how often they come up:\n"
422
+ "\n"
423
+ " A small set to iterate on. --fraction 0.05 gives you something that\n"
424
+ " trains in minutes while you get the pipeline right.\n"
425
+ "\n"
426
+ " Levelling an imbalanced dataset. --per-class 500 caps the dominant\n"
427
+ " class without touching the rare ones, which is usually a better\n"
428
+ " starting point than class weights.\n"
429
+ "\n"
430
+ " A review sample. A stratified 200 images is enough to eyeball label\n"
431
+ " quality across every class.\n"
432
+ "\n"
433
+ "Always seeded, so 'the 500 images we evaluated on' stays meaningful."
434
+ ),
435
+ )
436
+ def sample(params: dict[str, Any], ctx: Context) -> OpResult:
437
+ result = OpResult(op="dataset.sample", dry_run=ctx.dry_run)
438
+ folder: Path = params["folder"]
439
+ output: Path = params["output"]
440
+ if output.resolve() == folder.resolve():
441
+ result.fail("output must differ from folder")
442
+ return result
443
+
444
+ try:
445
+ dataset = formats.read_dataset(
446
+ folder, params["source_format"], recursive=params["recursive"], ctx=ctx
447
+ )
448
+ except AnnotationError as exc:
449
+ result.fail(str(exc))
450
+ return result
451
+ result.warnings.extend(dataset.warnings)
452
+
453
+ items = list(dataset.items)
454
+ if not items:
455
+ result.fail(f"no images found in {folder}")
456
+ return result
457
+
458
+ rng = random.Random(params["seed"])
459
+ if params["per_class"]:
460
+ chosen = _cap_per_class(items, params["per_class"], rng)
461
+ else:
462
+ wanted = params["count"] or max(1, round(len(items) * params["fraction"]))
463
+ wanted = min(wanted, len(items))
464
+ chosen = (
465
+ _stratified(items, wanted, rng) if params["stratify"] else rng.sample(items, wanted)
466
+ )
467
+
468
+ # Sorting restores a stable on-disk order after the random selection, so
469
+ # the output folder listing does not look shuffled.
470
+ chosen.sort(key=lambda i: i.stem)
471
+
472
+ target_format = params["format"]
473
+ if target_format == "same":
474
+ target_format = dataset.source_format
475
+ writer = DatasetWriter(
476
+ output, target_format, ctx, classes=dataset.labels, quality=params["quality"]
477
+ )
478
+
479
+ per_class: Counter[str] = Counter()
480
+ for position, item in enumerate(chosen, start=1):
481
+ ctx.check_cancelled()
482
+ ctx.emit(position, len(chosen), item.stem)
483
+ per_class.update(item.annotation.label_set)
484
+ writer.add(
485
+ item.annotation,
486
+ stem=item.stem,
487
+ source=item.image,
488
+ split=item.split,
489
+ suffix=item.image.suffix if item.image else ".jpg",
490
+ )
491
+
492
+ report = writer.close()
493
+ result.actions.extend(writer.actions)
494
+ result.counts.update(
495
+ {
496
+ "available": len(items),
497
+ "sampled": len(chosen),
498
+ "objects": sum(len(i.annotation.shapes) for i in chosen),
499
+ }
500
+ )
501
+ result.warnings.extend(report["warnings"])
502
+ result.data.update(
503
+ {
504
+ "output": output,
505
+ "format": target_format,
506
+ "seed": params["seed"],
507
+ "images_per_class": dict(sorted(per_class.items())),
508
+ }
509
+ )
510
+ return result
511
+
512
+
513
+ def _stratified(items: list[formats.Item], wanted: int, rng: random.Random) -> list[formats.Item]:
514
+ """Sample proportionally per class signature, so rare classes survive.
515
+
516
+ Grouping on the *set* of classes in an image, rather than on a single
517
+ class, is what keeps multi-label images from being counted twice.
518
+ """
519
+ groups: dict[str, list[formats.Item]] = {}
520
+ for item in items:
521
+ key = ",".join(sorted(item.annotation.label_set)) or "(background)"
522
+ groups.setdefault(key, []).append(item)
523
+
524
+ chosen: list[formats.Item] = []
525
+ remainders: list[tuple[float, str]] = []
526
+ for key, members in sorted(groups.items()):
527
+ share = len(members) / len(items) * wanted
528
+ take = int(share)
529
+ rng.shuffle(members)
530
+ chosen.extend(members[:take])
531
+ remainders.append((share - take, key))
532
+
533
+ # Largest-remainder, so the total lands exactly on `wanted` rather than a
534
+ # few short from repeated rounding down.
535
+ remainders.sort(reverse=True)
536
+ index = 0
537
+ while len(chosen) < wanted and index < len(remainders):
538
+ key = remainders[index][1]
539
+ members = groups[key]
540
+ already = sum(1 for c in chosen if c in members)
541
+ if already < len(members):
542
+ chosen.append(members[already])
543
+ index += 1
544
+ return chosen[:wanted]
545
+
546
+
547
+ def _cap_per_class(
548
+ items: list[formats.Item], cap: int, rng: random.Random
549
+ ) -> list[formats.Item]:
550
+ """Keep images until every class has reached ``cap``.
551
+
552
+ Rarest class first: an image showing both a common and a rare class should
553
+ be admitted on the rare class's account, or the rare class runs out of
554
+ candidates while the common one is still filling up.
555
+ """
556
+ frequency: Counter[str] = Counter()
557
+ for item in items:
558
+ frequency.update(item.annotation.label_set)
559
+
560
+ order = sorted(
561
+ items,
562
+ key=lambda i: (
563
+ min((frequency[label] for label in i.annotation.label_set), default=10**9),
564
+ i.stem,
565
+ ),
566
+ )
567
+ rng.shuffle(order)
568
+ order.sort(
569
+ key=lambda i: min((frequency[label] for label in i.annotation.label_set), default=10**9)
570
+ )
571
+
572
+ taken: Counter[str] = Counter()
573
+ chosen: list[formats.Item] = []
574
+ for item in order:
575
+ labels = item.annotation.label_set
576
+ if not labels:
577
+ continue
578
+ if all(taken[label] >= cap for label in labels):
579
+ continue
580
+ chosen.append(item)
581
+ taken.update(labels)
582
+ return chosen