easyannotate 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. data_factory/__init__.py +82 -0
  2. data_factory/__main__.py +6 -0
  3. data_factory/annotate/__init__.py +15 -0
  4. data_factory/annotate/attach.py +398 -0
  5. data_factory/annotate/demo.py +161 -0
  6. data_factory/annotate/images.py +443 -0
  7. data_factory/annotate/labels.py +399 -0
  8. data_factory/annotate/loop.py +359 -0
  9. data_factory/annotate/projects.py +207 -0
  10. data_factory/annotate/review.py +471 -0
  11. data_factory/annotate/schema.sql +171 -0
  12. data_factory/annotate/shapes.py +488 -0
  13. data_factory/annotate/store.py +292 -0
  14. data_factory/annotate/tools/__init__.py +39 -0
  15. data_factory/annotate/tools/fake.py +170 -0
  16. data_factory/annotate/tools/install.py +254 -0
  17. data_factory/annotate/tools/local.py +389 -0
  18. data_factory/annotate/tools/postprocess.py +175 -0
  19. data_factory/annotate/tools/protocol.py +238 -0
  20. data_factory/annotate/tools/registry.py +196 -0
  21. data_factory/annotate/tools/results.py +83 -0
  22. data_factory/annotate/tools/serve.py +106 -0
  23. data_factory/cli/__init__.py +21 -0
  24. data_factory/cli/main.py +837 -0
  25. data_factory/cli/menu.py +173 -0
  26. data_factory/cli/render.py +158 -0
  27. data_factory/core/__init__.py +11 -0
  28. data_factory/core/annotations/__init__.py +5 -0
  29. data_factory/core/annotations/coco.py +394 -0
  30. data_factory/core/annotations/csvfmt.py +174 -0
  31. data_factory/core/annotations/dota.py +98 -0
  32. data_factory/core/annotations/formats.py +626 -0
  33. data_factory/core/annotations/labelme.py +153 -0
  34. data_factory/core/annotations/model.py +334 -0
  35. data_factory/core/annotations/voc.py +231 -0
  36. data_factory/core/annotations/yolo.py +283 -0
  37. data_factory/core/bktree.py +89 -0
  38. data_factory/core/fs.py +245 -0
  39. data_factory/core/imaging.py +626 -0
  40. data_factory/core/ops/__init__.py +44 -0
  41. data_factory/core/ops/analyze.py +383 -0
  42. data_factory/core/ops/annotate_export.py +326 -0
  43. data_factory/core/ops/audit.py +274 -0
  44. data_factory/core/ops/augment.py +454 -0
  45. data_factory/core/ops/collect.py +241 -0
  46. data_factory/core/ops/combine.py +582 -0
  47. data_factory/core/ops/convert.py +212 -0
  48. data_factory/core/ops/dataset.py +246 -0
  49. data_factory/core/ops/dedup.py +235 -0
  50. data_factory/core/ops/export.py +168 -0
  51. data_factory/core/ops/image.py +407 -0
  52. data_factory/core/ops/labels.py +215 -0
  53. data_factory/core/ops/pairing.py +133 -0
  54. data_factory/core/ops/rename.py +154 -0
  55. data_factory/core/ops/report.py +323 -0
  56. data_factory/core/ops/visualize.py +164 -0
  57. data_factory/core/pairs.py +144 -0
  58. data_factory/core/parallel.py +191 -0
  59. data_factory/core/provenance.py +136 -0
  60. data_factory/core/registry.py +339 -0
  61. data_factory/core/stats.py +403 -0
  62. data_factory/core/types.py +148 -0
  63. data_factory/core/writer.py +194 -0
  64. data_factory/desktop.py +207 -0
  65. data_factory/hardware.py +243 -0
  66. data_factory/pipeline.py +192 -0
  67. data_factory/py.typed +0 -0
  68. data_factory/service/__init__.py +26 -0
  69. data_factory/service/app.py +483 -0
  70. data_factory/service/binding.py +174 -0
  71. data_factory/service/deps.py +133 -0
  72. data_factory/service/jobs.py +177 -0
  73. data_factory/service/limits.py +71 -0
  74. data_factory/service/routers/__init__.py +11 -0
  75. data_factory/service/routers/annotate.py +582 -0
  76. data_factory/service/routers/local.py +464 -0
  77. data_factory/service/routers/media.py +154 -0
  78. data_factory/service/routers/predictions.py +116 -0
  79. data_factory/service/routers/promote.py +182 -0
  80. data_factory/service/routers/tools.py +308 -0
  81. data_factory/service/static/annotate/api.js +246 -0
  82. data_factory/service/static/annotate/app.js +1696 -0
  83. data_factory/service/static/annotate/assist.js +106 -0
  84. data_factory/service/static/annotate/filmstrip.js +286 -0
  85. data_factory/service/static/annotate/hit.js +146 -0
  86. data_factory/service/static/annotate/labels.js +197 -0
  87. data_factory/service/static/annotate/render.js +286 -0
  88. data_factory/service/static/annotate/scene.js +277 -0
  89. data_factory/service/static/annotate/shapelist.js +72 -0
  90. data_factory/service/static/annotate/tools/box.js +96 -0
  91. data_factory/service/static/annotate/tools/line.js +96 -0
  92. data_factory/service/static/annotate/tools/obb.js +88 -0
  93. data_factory/service/static/annotate/tools/point.js +49 -0
  94. data_factory/service/static/annotate/tools/polygon.js +110 -0
  95. data_factory/service/static/annotate/tools/select.js +227 -0
  96. data_factory/service/static/annotate/undo.js +176 -0
  97. data_factory/service/static/annotate/viewport.js +156 -0
  98. data_factory/service/static/annotate.html +390 -0
  99. data_factory/service/static/index.html +632 -0
  100. data_factory/service/workspace.py +244 -0
  101. easyannotate/__init__.py +64 -0
  102. easyannotate/__main__.py +5 -0
  103. easyannotate/project.py +181 -0
  104. easyannotate/py.typed +0 -0
  105. easyannotate-0.4.0.dist-info/METADATA +767 -0
  106. easyannotate-0.4.0.dist-info/RECORD +109 -0
  107. easyannotate-0.4.0.dist-info/WHEEL +4 -0
  108. easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
  109. easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,626 @@
1
+ """One dataset, five formats, read and written through a single interface.
2
+
3
+ This is the module that turns a pile of readers and writers into a converter.
4
+ Because everything meets in the middle at :class:`Annotation`, supporting N
5
+ formats costs N readers and N writers -- not the N-squared converters a
6
+ pairwise design would need. Adding CreateML tomorrow means one file, and it
7
+ immediately converts to and from all four existing formats.
8
+
9
+ The layouts recognised are the ones exporters actually produce, not the ones
10
+ the specifications describe:
11
+
12
+ voc ``x.jpg`` + ``x.xml`` side by side, or Annotations/ + JPEGImages/
13
+ yolo ``images/`` + ``labels/`` + ``data.yaml``, or flat .txt sidecars
14
+ coco ``_annotations.coco.json``, or ``annotations/instances_*.json``
15
+ labelme ``x.jpg`` + ``x.json`` side by side
16
+ csv a single ``_annotations.csv``
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ from collections.abc import Iterable
22
+ from dataclasses import dataclass, field
23
+ from pathlib import Path
24
+
25
+ from .. import fs, parallel
26
+ from ..pairs import PairIndex
27
+ from ..types import Action, Context
28
+ from . import coco as coco_fmt
29
+ from . import csvfmt, labelme, voc, yolo
30
+ from . import dota
31
+ from .model import Annotation, AnnotationError
32
+
33
+ FORMATS = ("voc", "yolo", "coco", "labelme", "csv", "dota")
34
+
35
+ #: Formats able to carry a polygon outline. Converting out of one of these into
36
+ #: a box-only format is lossy, and the converter says so rather than quietly
37
+ #: flattening the masks.
38
+ SEGMENTATION_FORMATS = frozenset({"voc", "yolo", "coco", "labelme"})
39
+
40
+
41
+ @dataclass(slots=True)
42
+ class Item:
43
+ """One image and what is known about it."""
44
+
45
+ annotation: Annotation
46
+ image: Path | None = None
47
+ split: str = ""
48
+
49
+ @property
50
+ def stem(self) -> str:
51
+ source = self.image.stem if self.image else Path(self.annotation.filename).stem
52
+ return source
53
+
54
+
55
+ @dataclass
56
+ class Dataset:
57
+ """A whole dataset, loaded into the neutral model."""
58
+
59
+ items: list[Item] = field(default_factory=list)
60
+ source_format: str = ""
61
+ classes: list[str] = field(default_factory=list)
62
+ warnings: list[str] = field(default_factory=list)
63
+
64
+ @property
65
+ def labels(self) -> list[str]:
66
+ if self.classes:
67
+ return self.classes
68
+ found: set[str] = set()
69
+ for item in self.items:
70
+ found |= item.annotation.label_set
71
+ return sorted(found)
72
+
73
+ def stats(self) -> dict[str, int]:
74
+ objects = sum(len(i.annotation.shapes) for i in self.items)
75
+ return {
76
+ "images": len(self.items),
77
+ "objects": objects,
78
+ "classes": len(self.labels),
79
+ "backgrounds": sum(1 for i in self.items if i.annotation.is_background),
80
+ "with_polygons": sum(1 for i in self.items if i.annotation.has_polygons),
81
+ "missing_images": sum(1 for i in self.items if i.image is None),
82
+ }
83
+
84
+
85
+ # ------------------------------------------------------------- detection
86
+
87
+ def detect(folder: Path) -> str:
88
+ """Guess the format of a folder by looking for each layout's marker.
89
+
90
+ Ordered by how distinctive the evidence is: a ``data.yaml`` or a COCO json
91
+ is conclusive, while loose ``.json`` files need their contents inspected to
92
+ tell LabelMe from COCO.
93
+ """
94
+ folder = Path(folder)
95
+ if yolo.find_data_yaml(folder):
96
+ return "yolo"
97
+ if (folder / yolo.LABEL_DIR).is_dir():
98
+ # DOTA uses the same images/ + labels/ layout; only the lines differ.
99
+ return "dota" if _looks_like_dota(folder) else "yolo"
100
+
101
+ for path in coco_fmt.find_annotation_files(folder):
102
+ if _looks_like_coco(path):
103
+ return "coco"
104
+
105
+ counts = {
106
+ "voc": _count(folder, ".xml"),
107
+ "labelme": _count(folder, ".json"),
108
+ "yolo": _count(folder, ".txt"),
109
+ "csv": _count(folder, ".csv"),
110
+ }
111
+ if counts["voc"]:
112
+ return "voc"
113
+ if counts["labelme"]:
114
+ return "labelme"
115
+ if counts["csv"]:
116
+ return "csv"
117
+ if counts["yolo"]:
118
+ # Both are bare .txt files; DOTA names its class in the ninth field,
119
+ # where YOLO has nothing but numbers.
120
+ return "dota" if _looks_like_dota(folder) else "yolo"
121
+ raise AnnotationError(
122
+ f"cannot tell what format {folder} is -- found no .xml, .txt, .json or "
123
+ f".csv labels. Pass the format explicitly."
124
+ )
125
+
126
+
127
+ def _looks_like_dota(folder: Path) -> bool:
128
+ for path in folder.rglob("*.txt"):
129
+ if not path.is_file() or path.name == "classes.txt":
130
+ continue
131
+ try:
132
+ lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
133
+ except OSError:
134
+ continue
135
+ for line in lines:
136
+ parts = line.split()
137
+ if not parts or line.startswith(dota.HEADER_PREFIXES):
138
+ continue
139
+ if len(parts) < 9:
140
+ return False
141
+ try:
142
+ float(parts[8])
143
+ except ValueError:
144
+ return True
145
+ return False
146
+ return False
147
+
148
+
149
+ def _count(folder: Path, suffix: str, limit: int = 1) -> int:
150
+ seen = 0
151
+ for path in folder.rglob(f"*{suffix}"):
152
+ if path.is_file() and path.name != "data.yaml":
153
+ seen += 1
154
+ if seen >= limit:
155
+ return seen
156
+ return seen
157
+
158
+
159
+ def _looks_like_coco(path: Path) -> bool:
160
+ """Peek at the head of the file rather than parsing a 200MB json."""
161
+ try:
162
+ with path.open("r", encoding="utf-8") as handle:
163
+ head = handle.read(4096)
164
+ except OSError:
165
+ return False
166
+ return '"categories"' in head or '"annotations"' in head
167
+
168
+
169
+ # ---------------------------------------------------------------- reading
170
+
171
+ def read_dataset(
172
+ folder: Path,
173
+ fmt: str = "auto",
174
+ *,
175
+ recursive: bool = True,
176
+ ctx: Context | None = None,
177
+ ) -> Dataset:
178
+ """Load a folder into the neutral model, whatever format it is in."""
179
+ folder = Path(folder)
180
+ resolved = detect(folder) if fmt in ("", "auto") else fmt
181
+ if resolved not in FORMATS:
182
+ raise AnnotationError(f"unknown format {resolved!r}; known: {', '.join(FORMATS)}")
183
+
184
+ reader = {
185
+ "voc": _read_voc,
186
+ "yolo": _read_yolo,
187
+ "coco": _read_coco,
188
+ "labelme": _read_labelme,
189
+ "csv": _read_csv,
190
+ "dota": _read_dota,
191
+ }[resolved]
192
+ dataset = reader(folder, recursive, ctx or Context())
193
+ dataset.source_format = resolved
194
+ return dataset
195
+
196
+
197
+ def _read_sidecars(
198
+ folder: Path, recursive: bool, ext: str, parse
199
+ ) -> Dataset:
200
+ """Shared body for the two sidecar formats (VOC and LabelMe)."""
201
+ dataset = Dataset()
202
+ index = PairIndex.scan(folder, recursive=recursive, label_ext=ext)
203
+ for pair in index:
204
+ if pair.label is None:
205
+ if pair.image is not None:
206
+ # An unlabelled image is a real dataset member; carry it as a
207
+ # background rather than dropping it on the floor.
208
+ dataset.items.append(
209
+ Item(annotation=_empty_annotation(pair.image), image=pair.image)
210
+ )
211
+ continue
212
+ try:
213
+ annotation = parse(pair.label)
214
+ except AnnotationError as exc:
215
+ dataset.warnings.append(str(exc))
216
+ continue
217
+ if pair.image is not None and (annotation.width <= 0 or annotation.height <= 0):
218
+ annotation.width, annotation.height = _size_of(pair.image, dataset)
219
+ dataset.items.append(Item(annotation=annotation, image=pair.image, split=_split_of(pair.key)))
220
+ return dataset
221
+
222
+
223
+ def _read_voc(folder: Path, recursive: bool, ctx: Context) -> Dataset:
224
+ return _read_sidecars(folder, recursive, ".xml", voc.read)
225
+
226
+
227
+ def _read_labelme(folder: Path, recursive: bool, ctx: Context) -> Dataset:
228
+ skipped: list[str] = []
229
+
230
+ def parse(path: Path) -> Annotation:
231
+ return labelme.read(path, skipped_out=skipped)
232
+
233
+ dataset = _read_sidecars(folder, recursive, ".json", parse)
234
+ if skipped:
235
+ dataset.warnings.append(
236
+ f"LabelMe shape types not representable as polygons were skipped: "
237
+ f"{', '.join(sorted(set(skipped)))}"
238
+ )
239
+ return dataset
240
+
241
+
242
+ def _read_yolo(folder: Path, recursive: bool, ctx: Context) -> Dataset:
243
+ dataset = Dataset()
244
+ yaml_path = yolo.find_data_yaml(folder)
245
+ if yaml_path is not None:
246
+ classes, _ = yolo.read_data_yaml(yaml_path)
247
+ dataset.classes = list(classes.names)
248
+ else:
249
+ # Without a class list the indices are meaningless, so say so loudly
250
+ # and fall back to placeholder names rather than inventing a mapping.
251
+ classes = yolo.ClassList([])
252
+ dataset.warnings.append(
253
+ "no data.yaml found -- classes will be named class_0, class_1, ... "
254
+ "Point --classes at the right file to keep the real names."
255
+ )
256
+
257
+ # YOLO stores coordinates normalised, so every label needs its image's
258
+ # dimensions before it can be read at all -- one header read per image.
259
+ # Serial, that is the dominant cost of loading a large YOLO dataset.
260
+ images = list(fs.iter_images(folder, recursive=recursive))
261
+ sizes = dict(
262
+ parallel.collect(
263
+ images,
264
+ _image_size,
265
+ ctx=ctx,
266
+ label=lambda image: image.name,
267
+ on_error=lambda image, exc: dataset.warnings.append(
268
+ f"{image.name}: cannot read image size ({exc})"
269
+ ),
270
+ )
271
+ )
272
+
273
+ for image in images:
274
+ label_path = yolo.label_path_for(image, folder)
275
+ width, height = sizes.get(image, (0, 0))
276
+ if width <= 0:
277
+ continue
278
+ if not label_path.is_file():
279
+ dataset.items.append(
280
+ Item(annotation=Annotation(filename=image.name, width=width, height=height),
281
+ image=image)
282
+ )
283
+ continue
284
+ try:
285
+ annotation = yolo.read(
286
+ label_path, classes, width, height, filename=image.name
287
+ )
288
+ except AnnotationError as exc:
289
+ dataset.warnings.append(str(exc))
290
+ continue
291
+ dataset.items.append(
292
+ Item(
293
+ annotation=annotation,
294
+ image=image,
295
+ split=_split_of(str(fs.relative_to_safe(image, folder)).replace("\\", "/")),
296
+ )
297
+ )
298
+ if not dataset.classes:
299
+ dataset.classes = list(classes.names)
300
+ return dataset
301
+
302
+
303
+ def _read_dota(folder: Path, recursive: bool, ctx: Context) -> Dataset:
304
+ """Like YOLO -- one .txt beside or under labels/ per image -- but the class
305
+ is named in the file, so there is no list to find."""
306
+ dataset = Dataset()
307
+ images = list(fs.iter_images(folder, recursive=recursive))
308
+ sizes = dict(
309
+ parallel.collect(
310
+ images,
311
+ _image_size,
312
+ ctx=ctx,
313
+ label=lambda image: image.name,
314
+ on_error=lambda image, exc: dataset.warnings.append(
315
+ f"{image.name}: cannot read image size ({exc})"
316
+ ),
317
+ )
318
+ )
319
+ for image in images:
320
+ label_path = yolo.label_path_for(image, folder)
321
+ width, height = sizes.get(image, (0, 0))
322
+ if width <= 0:
323
+ continue
324
+ if not label_path.is_file():
325
+ dataset.items.append(
326
+ Item(annotation=Annotation(filename=image.name, width=width, height=height),
327
+ image=image)
328
+ )
329
+ continue
330
+ try:
331
+ annotation = dota.read(label_path, width, height, filename=image.name)
332
+ except AnnotationError as exc:
333
+ dataset.warnings.append(str(exc))
334
+ continue
335
+ dataset.items.append(
336
+ Item(
337
+ annotation=annotation,
338
+ image=image,
339
+ split=_split_of(str(fs.relative_to_safe(image, folder)).replace("\\", "/")),
340
+ )
341
+ )
342
+ return dataset
343
+
344
+
345
+ def _read_coco(folder: Path, recursive: bool, ctx: Context) -> Dataset:
346
+ dataset = Dataset()
347
+ files = [p for p in coco_fmt.find_annotation_files(folder) if _looks_like_coco(p)]
348
+ if not files:
349
+ raise AnnotationError(f"no COCO annotation json found under {folder}")
350
+
351
+ images_by_name = _image_index(folder)
352
+ unsupported: list[str] = []
353
+ for path in files:
354
+ document = coco_fmt.load(path)
355
+ split = _split_of(path.stem)
356
+ for annotation in coco_fmt.to_annotations(document, unsupported_out=unsupported):
357
+ image = _pick_image(images_by_name, annotation.filename, split)
358
+ dataset.items.append(Item(annotation=annotation, image=image, split=split))
359
+ for category in document.get("categories", []):
360
+ name = str(category.get("name", ""))
361
+ if name and name not in dataset.classes:
362
+ dataset.classes.append(name)
363
+ dataset.warnings.extend(unsupported)
364
+ return dataset
365
+
366
+
367
+ def _read_csv(folder: Path, recursive: bool, ctx: Context) -> Dataset:
368
+ dataset = Dataset()
369
+ candidates = sorted(folder.rglob("*.csv")) if recursive else sorted(folder.glob("*.csv"))
370
+ if not candidates:
371
+ raise AnnotationError(f"no .csv annotation file found under {folder}")
372
+
373
+ images_by_name = _image_index(folder)
374
+ for path in candidates:
375
+ for annotation in csvfmt.read(path):
376
+ image = _pick_image(images_by_name, annotation.filename)
377
+ if image is not None and annotation.width <= 0:
378
+ annotation.width, annotation.height = _size_of(image, dataset)
379
+ dataset.items.append(Item(annotation=annotation, image=image))
380
+ return dataset
381
+
382
+
383
+ def _image_index(folder: Path) -> dict[str, list[Path]]:
384
+ """Every image under a folder, findable by both name and stem.
385
+
386
+ COCO and CSV reference images by name only, so the file could be anywhere
387
+ in the tree -- and in a split dataset ``a.jpg`` legitimately exists in both
388
+ ``train/`` and ``val/``. Every candidate is kept so the caller can pick the
389
+ one in the right split rather than whichever sorted first.
390
+ """
391
+ index: dict[str, list[Path]] = {}
392
+ for image in fs.iter_images(folder, recursive=True):
393
+ index.setdefault(image.name, []).append(image)
394
+ if image.stem != image.name:
395
+ index.setdefault(image.stem, []).append(image)
396
+ return index
397
+
398
+
399
+ def _pick_image(
400
+ index: dict[str, list[Path]], filename: str, split: str = ""
401
+ ) -> Path | None:
402
+ """Resolve a referenced filename to an actual file, preferring its split.
403
+
404
+ Without the split preference, a COCO dataset whose train and val folders
405
+ share a filename would attach val's annotations to train's image -- wrong
406
+ in a way nothing downstream could detect.
407
+ """
408
+ candidates = index.get(filename) or index.get(Path(filename).name) or index.get(
409
+ Path(filename).stem
410
+ )
411
+ if not candidates:
412
+ return None
413
+ if split:
414
+ for candidate in candidates:
415
+ if split in {part.lower() for part in candidate.parts}:
416
+ return candidate
417
+ return candidates[0]
418
+
419
+
420
+ def _empty_annotation(image: Path) -> Annotation:
421
+ try:
422
+ width, height = _image_size(image)
423
+ except Exception:
424
+ width, height = 0, 0
425
+ return Annotation(filename=image.name, width=width, height=height, shapes=[])
426
+
427
+
428
+ def _size_of(image: Path, dataset: Dataset) -> tuple[int, int]:
429
+ try:
430
+ return _image_size(image)
431
+ except Exception as exc:
432
+ dataset.warnings.append(f"{image.name}: cannot read image size ({exc})")
433
+ return 0, 0
434
+
435
+
436
+ def _image_size(path: Path) -> tuple[int, int]:
437
+ from ..imaging import image_size
438
+
439
+ return image_size(path)
440
+
441
+
442
+ def _split_of(key: str) -> str:
443
+ """Recover train/val/test from a path or filename, when it is there."""
444
+ lowered = key.lower().replace("\\", "/")
445
+ for name in ("train", "valid", "val", "test"):
446
+ if f"/{name}/" in f"/{lowered}/" or lowered.endswith(f"_{name}"):
447
+ return "val" if name == "valid" else name
448
+ return ""
449
+
450
+
451
+ # ---------------------------------------------------------------- writing
452
+
453
+ def write_dataset(
454
+ dataset: Dataset,
455
+ output: Path,
456
+ fmt: str,
457
+ *,
458
+ ctx: Context,
459
+ copy_images: bool = True,
460
+ segmentation: bool = True,
461
+ on_action=None,
462
+ ) -> dict[str, object]:
463
+ """Serialise a dataset into ``fmt`` under ``output``.
464
+
465
+ Image files are copied through the usual :func:`fs.perform` choke point, so
466
+ ``--dry-run`` reports the entire conversion without writing a byte. Label
467
+ files are written directly, but only ever inside ``output``, and never at
468
+ all in dry-run mode.
469
+ """
470
+ if fmt not in FORMATS:
471
+ raise AnnotationError(f"unknown format {fmt!r}; known: {', '.join(FORMATS)}")
472
+
473
+ output = Path(output)
474
+ report: dict[str, object] = {"format": fmt, "written": 0, "images": 0, "warnings": []}
475
+ warnings: list[str] = report["warnings"] # type: ignore[assignment]
476
+
477
+ if segmentation and fmt not in SEGMENTATION_FORMATS:
478
+ polygons = sum(1 for i in dataset.items if i.annotation.has_polygons)
479
+ if polygons:
480
+ warnings.append(
481
+ f"{fmt} cannot store outlines: {polygons} image(s) keep their boxes "
482
+ f"but lose their polygons"
483
+ )
484
+
485
+ writer = {
486
+ "voc": _write_voc,
487
+ "yolo": _write_yolo,
488
+ "coco": _write_coco,
489
+ "labelme": _write_labelme,
490
+ "csv": _write_csv,
491
+ "dota": _write_dota,
492
+ }[fmt]
493
+ writer(dataset, output, ctx, report, copy_images, segmentation, on_action)
494
+ return report
495
+
496
+
497
+ def _copy_image(
498
+ item: Item, destination: Path, ctx: Context, report: dict, on_action
499
+ ) -> None:
500
+ if item.image is None or not destination:
501
+ return
502
+ action = fs.FileAction(Action.COPY, item.image, destination, "convert")
503
+ fs.perform(action, ctx)
504
+ report["images"] = int(report["images"]) + 1
505
+ if on_action is not None:
506
+ on_action(action)
507
+
508
+
509
+ def _split_dir(base: Path, item: Item) -> Path:
510
+ return base / item.split if item.split else base
511
+
512
+
513
+ def _write_voc(dataset, output, ctx, report, copy_images, segmentation, on_action) -> None:
514
+ for item in dataset.items:
515
+ folder = _split_dir(output, item)
516
+ fs.ensure_dir(folder, ctx)
517
+ stem = item.stem
518
+ if copy_images and item.image is not None:
519
+ _copy_image(item, folder / f"{stem}{item.image.suffix}", ctx, report, on_action)
520
+ if not ctx.dry_run:
521
+ voc.write(item.annotation, folder / f"{stem}.xml")
522
+ report["written"] = int(report["written"]) + 1
523
+
524
+
525
+ def _write_labelme(dataset, output, ctx, report, copy_images, segmentation, on_action) -> None:
526
+ for item in dataset.items:
527
+ folder = _split_dir(output, item)
528
+ fs.ensure_dir(folder, ctx)
529
+ stem = item.stem
530
+ if copy_images and item.image is not None:
531
+ _copy_image(item, folder / f"{stem}{item.image.suffix}", ctx, report, on_action)
532
+ if not ctx.dry_run:
533
+ labelme.write(item.annotation, folder / f"{stem}.json")
534
+ report["written"] = int(report["written"]) + 1
535
+
536
+
537
+ def _write_yolo(dataset, output, ctx, report, copy_images, segmentation, on_action) -> None:
538
+ # Seed the class list from the dataset so indices follow the declared order
539
+ # rather than whichever image happened to be processed first.
540
+ classes = yolo.ClassList(list(dataset.labels))
541
+ splits: dict[str, str] = {}
542
+
543
+ for item in dataset.items:
544
+ split = item.split or "train"
545
+ splits.setdefault(split, f"{yolo.IMAGE_DIR}/{split}")
546
+ image_dir = output / yolo.IMAGE_DIR / split
547
+ label_dir = output / yolo.LABEL_DIR / split
548
+ fs.ensure_dir(image_dir, ctx)
549
+ fs.ensure_dir(label_dir, ctx)
550
+ stem = item.stem
551
+ if copy_images and item.image is not None:
552
+ _copy_image(item, image_dir / f"{stem}{item.image.suffix}", ctx, report, on_action)
553
+ if not ctx.dry_run:
554
+ yolo.write(item.annotation, label_dir / f"{stem}.txt", classes, segmentation=segmentation)
555
+ report["written"] = int(report["written"]) + 1
556
+
557
+ if not ctx.dry_run:
558
+ yolo.write_data_yaml(output / yolo.DATA_YAML, classes, splits=splits, root=".")
559
+ report["classes"] = list(classes.names)
560
+
561
+
562
+ def _write_dota(dataset, output, ctx, report, copy_images, segmentation, on_action) -> None:
563
+ for item in dataset.items:
564
+ split = item.split or "train"
565
+ image_dir = output / yolo.IMAGE_DIR / split
566
+ label_dir = output / yolo.LABEL_DIR / split
567
+ fs.ensure_dir(image_dir, ctx)
568
+ fs.ensure_dir(label_dir, ctx)
569
+ stem = item.stem
570
+ if copy_images and item.image is not None:
571
+ _copy_image(item, image_dir / f"{stem}{item.image.suffix}", ctx, report, on_action)
572
+ if not ctx.dry_run:
573
+ dota.write(item.annotation, label_dir / f"{stem}.txt")
574
+ report["written"] = int(report["written"]) + 1
575
+ report["classes"] = list(dataset.labels)
576
+
577
+
578
+ def _write_coco(dataset, output, ctx, report, copy_images, segmentation, on_action) -> None:
579
+ by_split: dict[str, list[Item]] = {}
580
+ for item in dataset.items:
581
+ by_split.setdefault(item.split or "train", []).append(item)
582
+
583
+ for split, items in sorted(by_split.items()):
584
+ builder = coco_fmt.CocoBuilder(description=f"Data Factory export ({split})")
585
+ image_dir = output / split
586
+ fs.ensure_dir(image_dir, ctx)
587
+ for item in items:
588
+ name = f"{item.stem}{item.image.suffix if item.image else '.jpg'}"
589
+ builder.add(item.annotation, file_name=name)
590
+ if copy_images and item.image is not None:
591
+ _copy_image(item, image_dir / name, ctx, report, on_action)
592
+ report["written"] = int(report["written"]) + 1
593
+ if not ctx.dry_run:
594
+ fs.ensure_dir(output / "annotations", ctx)
595
+ builder.write(output / "annotations" / f"instances_{split}.json")
596
+ merges = builder.categories.merges()
597
+ if merges:
598
+ report["warnings"].append( # type: ignore[union-attr]
599
+ "case-variant classes merged: "
600
+ + "; ".join(f"{k} <- {', '.join(v)}" for k, v in merges.items())
601
+ )
602
+
603
+
604
+ def _write_csv(dataset, output, ctx, report, copy_images, segmentation, on_action) -> None:
605
+ fs.ensure_dir(output, ctx)
606
+ image_dir = output / "images"
607
+ annotations: list[Annotation] = []
608
+ for item in dataset.items:
609
+ annotations.append(item.annotation)
610
+ if copy_images and item.image is not None:
611
+ fs.ensure_dir(image_dir, ctx)
612
+ _copy_image(
613
+ item, image_dir / f"{item.stem}{item.image.suffix}", ctx, report, on_action
614
+ )
615
+ report["written"] = int(report["written"]) + 1
616
+ if not ctx.dry_run:
617
+ csvfmt.write(annotations, output / "_annotations.csv")
618
+
619
+
620
+ def label_extension(fmt: str) -> str:
621
+ """The sidecar extension a format uses, or empty for the collected ones."""
622
+ return {"voc": ".xml", "labelme": ".json", "yolo": ".txt", "dota": ".txt"}.get(fmt, "")
623
+
624
+
625
+ def iter_formats() -> Iterable[str]:
626
+ return FORMATS