easyannotate 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. data_factory/__init__.py +82 -0
  2. data_factory/__main__.py +6 -0
  3. data_factory/annotate/__init__.py +15 -0
  4. data_factory/annotate/attach.py +398 -0
  5. data_factory/annotate/demo.py +161 -0
  6. data_factory/annotate/images.py +443 -0
  7. data_factory/annotate/labels.py +399 -0
  8. data_factory/annotate/loop.py +359 -0
  9. data_factory/annotate/projects.py +207 -0
  10. data_factory/annotate/review.py +471 -0
  11. data_factory/annotate/schema.sql +171 -0
  12. data_factory/annotate/shapes.py +488 -0
  13. data_factory/annotate/store.py +292 -0
  14. data_factory/annotate/tools/__init__.py +39 -0
  15. data_factory/annotate/tools/fake.py +170 -0
  16. data_factory/annotate/tools/install.py +254 -0
  17. data_factory/annotate/tools/local.py +389 -0
  18. data_factory/annotate/tools/postprocess.py +175 -0
  19. data_factory/annotate/tools/protocol.py +238 -0
  20. data_factory/annotate/tools/registry.py +196 -0
  21. data_factory/annotate/tools/results.py +83 -0
  22. data_factory/annotate/tools/serve.py +106 -0
  23. data_factory/cli/__init__.py +21 -0
  24. data_factory/cli/main.py +837 -0
  25. data_factory/cli/menu.py +173 -0
  26. data_factory/cli/render.py +158 -0
  27. data_factory/core/__init__.py +11 -0
  28. data_factory/core/annotations/__init__.py +5 -0
  29. data_factory/core/annotations/coco.py +394 -0
  30. data_factory/core/annotations/csvfmt.py +174 -0
  31. data_factory/core/annotations/dota.py +98 -0
  32. data_factory/core/annotations/formats.py +626 -0
  33. data_factory/core/annotations/labelme.py +153 -0
  34. data_factory/core/annotations/model.py +334 -0
  35. data_factory/core/annotations/voc.py +231 -0
  36. data_factory/core/annotations/yolo.py +283 -0
  37. data_factory/core/bktree.py +89 -0
  38. data_factory/core/fs.py +245 -0
  39. data_factory/core/imaging.py +626 -0
  40. data_factory/core/ops/__init__.py +44 -0
  41. data_factory/core/ops/analyze.py +383 -0
  42. data_factory/core/ops/annotate_export.py +326 -0
  43. data_factory/core/ops/audit.py +274 -0
  44. data_factory/core/ops/augment.py +454 -0
  45. data_factory/core/ops/collect.py +241 -0
  46. data_factory/core/ops/combine.py +582 -0
  47. data_factory/core/ops/convert.py +212 -0
  48. data_factory/core/ops/dataset.py +246 -0
  49. data_factory/core/ops/dedup.py +235 -0
  50. data_factory/core/ops/export.py +168 -0
  51. data_factory/core/ops/image.py +407 -0
  52. data_factory/core/ops/labels.py +215 -0
  53. data_factory/core/ops/pairing.py +133 -0
  54. data_factory/core/ops/rename.py +154 -0
  55. data_factory/core/ops/report.py +323 -0
  56. data_factory/core/ops/visualize.py +164 -0
  57. data_factory/core/pairs.py +144 -0
  58. data_factory/core/parallel.py +191 -0
  59. data_factory/core/provenance.py +136 -0
  60. data_factory/core/registry.py +339 -0
  61. data_factory/core/stats.py +403 -0
  62. data_factory/core/types.py +148 -0
  63. data_factory/core/writer.py +194 -0
  64. data_factory/desktop.py +207 -0
  65. data_factory/hardware.py +243 -0
  66. data_factory/pipeline.py +192 -0
  67. data_factory/py.typed +0 -0
  68. data_factory/service/__init__.py +26 -0
  69. data_factory/service/app.py +483 -0
  70. data_factory/service/binding.py +174 -0
  71. data_factory/service/deps.py +133 -0
  72. data_factory/service/jobs.py +177 -0
  73. data_factory/service/limits.py +71 -0
  74. data_factory/service/routers/__init__.py +11 -0
  75. data_factory/service/routers/annotate.py +582 -0
  76. data_factory/service/routers/local.py +464 -0
  77. data_factory/service/routers/media.py +154 -0
  78. data_factory/service/routers/predictions.py +116 -0
  79. data_factory/service/routers/promote.py +182 -0
  80. data_factory/service/routers/tools.py +308 -0
  81. data_factory/service/static/annotate/api.js +246 -0
  82. data_factory/service/static/annotate/app.js +1696 -0
  83. data_factory/service/static/annotate/assist.js +106 -0
  84. data_factory/service/static/annotate/filmstrip.js +286 -0
  85. data_factory/service/static/annotate/hit.js +146 -0
  86. data_factory/service/static/annotate/labels.js +197 -0
  87. data_factory/service/static/annotate/render.js +286 -0
  88. data_factory/service/static/annotate/scene.js +277 -0
  89. data_factory/service/static/annotate/shapelist.js +72 -0
  90. data_factory/service/static/annotate/tools/box.js +96 -0
  91. data_factory/service/static/annotate/tools/line.js +96 -0
  92. data_factory/service/static/annotate/tools/obb.js +88 -0
  93. data_factory/service/static/annotate/tools/point.js +49 -0
  94. data_factory/service/static/annotate/tools/polygon.js +110 -0
  95. data_factory/service/static/annotate/tools/select.js +227 -0
  96. data_factory/service/static/annotate/undo.js +176 -0
  97. data_factory/service/static/annotate/viewport.js +156 -0
  98. data_factory/service/static/annotate.html +390 -0
  99. data_factory/service/static/index.html +632 -0
  100. data_factory/service/workspace.py +244 -0
  101. easyannotate/__init__.py +64 -0
  102. easyannotate/__main__.py +5 -0
  103. easyannotate/project.py +181 -0
  104. easyannotate/py.typed +0 -0
  105. easyannotate-0.4.0.dist-info/METADATA +767 -0
  106. easyannotate-0.4.0.dist-info/RECORD +109 -0
  107. easyannotate-0.4.0.dist-info/WHEEL +4 -0
  108. easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
  109. easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,168 @@
1
+ """Export a Pascal VOC folder to COCO.
2
+
3
+ Two behaviour changes from the original converter, both deliberate:
4
+
5
+ * Categories fold case by default, so ``Open`` and ``open`` are one class. The
6
+ original produced four classes for Project_Door V3 where there are two.
7
+ * ``area`` is the true polygon area when a mask exists, not the box area.
8
+
9
+ Both are reported in the result, so the export never changes your data silently.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ from .. import fs
18
+ from ..annotations import coco as coco_format
19
+ from ..annotations import voc
20
+ from ..annotations.model import AnnotationError
21
+ from ..pairs import PairIndex
22
+ from ..registry import Param, operation
23
+ from ..types import Action, Context, OpResult
24
+
25
+ PARAMS = (
26
+ Param("folder", "dir", "Folder of Pascal VOC image/label pairs.", required=True, must_exist=True),
27
+ Param("output", "dir", "Dataset root to create.", required=True),
28
+ Param("splits", "strs",
29
+ "Sub-folders to export as separate splits, e.g. train,val,test. Empty = one dataset."),
30
+ Param("copy_images", "bool", "Copy images into <output>/images.", default=True),
31
+ Param("case_policy", "choice", "Whether labels differing only by case are one class.",
32
+ default="casefold", choices=("casefold", "exact")),
33
+ Param("map", "mapping", "Class renames applied during export, as old=new pairs."),
34
+ Param("include_backgrounds", "bool",
35
+ "Include object-free images as hard negatives.", default=True),
36
+ Param("description", "str", "Description written into the COCO info block.",
37
+ default="Data Factory export"),
38
+ Param("recursive", "bool", "Scan sub-folders when not using splits.", default=False),
39
+ Param("label_ext", "str", "Sidecar label extension.", default=".xml"),
40
+ )
41
+
42
+
43
+ @operation(
44
+ "export.coco",
45
+ title="Export Pascal VOC to COCO",
46
+ category="export",
47
+ summary="Build instances.json (with segmentation where polygons exist) and gather the images.",
48
+ params=PARAMS,
49
+ notes=(
50
+ "Polygons from either annotation tool are exported as COCO segmentation --\n"
51
+ "including the SAM tool's <pt> dialect, which the old converter could not\n"
52
+ "read and silently exported as empty masks.\n\n"
53
+ "Run labels.list first: if it reports case variants, decide whether you want\n"
54
+ "case_policy=casefold (merge them) or to fix the labels at source with\n"
55
+ "labels.rename --fold-case."
56
+ ),
57
+ )
58
+ def to_coco(params: dict[str, Any], ctx: Context) -> OpResult:
59
+ result = OpResult(op="export.coco", dry_run=ctx.dry_run)
60
+ folder: Path = params["folder"]
61
+ output: Path = params["output"]
62
+ splits = list(params["splits"] or [])
63
+
64
+ targets: list[tuple[str, Path]]
65
+ if splits:
66
+ targets = [(name, folder / name) for name in splits]
67
+ missing = [name for name, path in targets if not path.is_dir()]
68
+ if missing:
69
+ result.fail(
70
+ f"split folder(s) not found under {folder}: {', '.join(missing)}. "
71
+ f"Present: {', '.join(sorted(p.name for p in folder.iterdir() if p.is_dir())) or 'none'}"
72
+ )
73
+ return result
74
+ else:
75
+ targets = [("", folder)]
76
+
77
+ for split_name, source in targets:
78
+ ctx.check_cancelled()
79
+ _export_one(split_name, source, output, params, result, ctx)
80
+
81
+ result.data["output"] = output
82
+ return result
83
+
84
+
85
+ def _export_one(
86
+ split_name: str,
87
+ source: Path,
88
+ output: Path,
89
+ params: dict[str, Any],
90
+ result: OpResult,
91
+ ctx: Context,
92
+ ) -> None:
93
+ prefix = f"{split_name}_" if split_name else ""
94
+ image_dir = output / (f"images/{split_name}" if split_name else "images")
95
+ ann_path = output / "annotations" / (f"instances_{split_name}.json" if split_name else "instances.json")
96
+
97
+ builder = coco_format.CocoBuilder(
98
+ description=params["description"],
99
+ case_policy=params["case_policy"],
100
+ label_map=params["map"] or {},
101
+ )
102
+
103
+ index = PairIndex.scan(
104
+ source,
105
+ recursive=params["recursive"] and not split_name,
106
+ label_ext=params["label_ext"],
107
+ )
108
+ pairs = index.complete()
109
+ result.counts[f"{prefix}scanned"] = len(index)
110
+
111
+ for position, pair in enumerate(pairs, start=1):
112
+ ctx.check_cancelled()
113
+ ctx.emit(position, len(pairs), f"{split_name or 'dataset'}: {pair.stem}")
114
+ assert pair.image is not None and pair.label is not None
115
+
116
+ try:
117
+ annotation = voc.read(pair.label)
118
+ except AnnotationError as exc:
119
+ result.warnings.append(f"{pair.label.name}: {exc}")
120
+ result.bump(f"{prefix}unreadable")
121
+ continue
122
+
123
+ if annotation.is_background and not params["include_backgrounds"]:
124
+ result.bump(f"{prefix}backgrounds_skipped")
125
+ continue
126
+
127
+ # The XML's <size> is not always trustworthy; fall back to the real
128
+ # image so COCO never records a zero-dimension entry.
129
+ if annotation.width <= 0 or annotation.height <= 0:
130
+ size = _image_size(pair.image)
131
+ if size is None:
132
+ result.warnings.append(f"{pair.image.name}: cannot determine image size, skipped")
133
+ result.bump(f"{prefix}unsized")
134
+ continue
135
+ annotation.width, annotation.height = size
136
+
137
+ builder.add(annotation, file_name=pair.image.name)
138
+
139
+ if params["copy_images"]:
140
+ destination = image_dir / pair.image.name
141
+ fs.perform(result.record(Action.COPY, pair.image, destination, "coco image"), ctx)
142
+ result.bump(f"{prefix}images_copied")
143
+
144
+ if not ctx.dry_run:
145
+ builder.write(ann_path)
146
+ result.record(Action.WRITE, source, ann_path, "coco annotations")
147
+
148
+ for key, value in builder.stats().items():
149
+ result.counts[f"{prefix}{key}"] = value
150
+
151
+ merges = builder.categories.merges()
152
+ if merges:
153
+ result.data.setdefault("category_merges", {})[split_name or "dataset"] = merges
154
+ result.warnings.append(
155
+ f"{split_name or 'dataset'}: merged case-variant classes -- "
156
+ + "; ".join(f"{name} <- {'/'.join(v)}" for name, v in merges.items())
157
+ )
158
+ result.data.setdefault("annotations_file", {})[split_name or "dataset"] = ann_path
159
+
160
+
161
+ def _image_size(path: Path) -> tuple[int, int] | None:
162
+ try:
163
+ from PIL import Image
164
+
165
+ with Image.open(path) as image:
166
+ return image.size
167
+ except Exception:
168
+ return None
@@ -0,0 +1,407 @@
1
+ """Image preprocessing that keeps the annotations correct.
2
+
3
+ Resizing a dataset is the step people most often do with a throwaway script,
4
+ and it is the step that most often corrupts a dataset -- because the boxes have
5
+ to be rescaled by exactly the same factor, and a letterboxed image needs the
6
+ padding offset added too. Doing it here means the geometry is handled once, by
7
+ :mod:`data_factory.core.imaging`, and every operation inherits it.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from pathlib import Path
13
+ from typing import Any
14
+
15
+ from .. import imaging, parallel
16
+ from ..annotations import formats
17
+ from ..annotations.model import AnnotationError
18
+ from ..registry import Param, operation
19
+ from ..types import Context, OpResult
20
+ from ..writer import DatasetWriter
21
+
22
+ #: ``same`` writes the dataset back out in the format it arrived in.
23
+ OUTPUT_FORMATS = ("same",) + formats.FORMATS
24
+ SOURCE_FORMATS = ("auto",) + formats.FORMATS
25
+
26
+ RESIZE_PARAMS = (
27
+ Param("folder", "dir", "Dataset to resize.", required=True, must_exist=True),
28
+ Param("output", "dir", "Where the resized dataset is written.", required=True),
29
+ Param("width", "int", "Target width in pixels.", required=True, minimum=1),
30
+ Param("height", "int", "Target height in pixels.", required=True, minimum=1),
31
+ Param(
32
+ "mode",
33
+ "choice",
34
+ "How to fit the image to the target size.",
35
+ default="letterbox",
36
+ choices=("letterbox", "stretch", "crop", "fit"),
37
+ ),
38
+ Param("source_format", "choice", "Input format.", default="auto", choices=SOURCE_FORMATS),
39
+ Param("format", "choice", "Output format.", default="same", choices=OUTPUT_FORMATS),
40
+ Param("quality", "int", "JPEG/WebP quality.", default=95, minimum=1, maximum=100),
41
+ Param("recursive", "bool", "Include sub-folders.", default=True),
42
+ Param("skip_larger", "bool", "Leave images already smaller than the target alone.", default=False),
43
+ Param("workers", "int", parallel.WORKERS_HELP, default=0, minimum=0, maximum=32),
44
+ )
45
+
46
+
47
+ @operation(
48
+ "image.resize",
49
+ title="Resize dataset",
50
+ category="image",
51
+ summary="Resize every image to a fixed size, rescaling boxes and polygons to match.",
52
+ params=RESIZE_PARAMS,
53
+ destructive=True,
54
+ notes=(
55
+ "Fit modes, and when each is right:\n"
56
+ " letterbox scale to fit, pad the rest. Nothing distorted, nothing\n"
57
+ " lost. The safe default, and what YOLO does internally.\n"
58
+ " stretch fill the frame exactly, distorting aspect ratio. Fine\n"
59
+ " when every source image already has one shape.\n"
60
+ " crop scale to fill, cut the overflow from the edges. Keeps\n"
61
+ " the aspect ratio but can cut objects at the border.\n"
62
+ " fit scale the long edge to fit, leave the short edge free.\n"
63
+ " Output sizes vary; use when the trainer handles that.\n"
64
+ "\n"
65
+ "Objects reduced to under two pixels are dropped, because a one-pixel\n"
66
+ "box is noise that a model will only learn to distrust."
67
+ ),
68
+ )
69
+ def resize(params: dict[str, Any], ctx: Context) -> OpResult:
70
+ result = OpResult(op="image.resize", dry_run=ctx.dry_run)
71
+ target_w, target_h = params["width"], params["height"]
72
+ mode = params["mode"]
73
+
74
+ def build(width: int, height: int):
75
+ if params["skip_larger"] and width <= target_w and height <= target_h:
76
+ return None
77
+ return imaging.make_resize(width, height, target_w, target_h, mode)
78
+
79
+ return _apply_geometry(
80
+ params, ctx, result, build, label=f"{target_w}x{target_h} ({mode})"
81
+ )
82
+
83
+
84
+ ORIENT_PARAMS = (
85
+ Param("folder", "dir", "Dataset to straighten.", required=True, must_exist=True),
86
+ Param("output", "dir", "Where the corrected dataset is written.", required=True),
87
+ Param("source_format", "choice", "Input format.", default="auto", choices=SOURCE_FORMATS),
88
+ Param("format", "choice", "Output format.", default="same", choices=OUTPUT_FORMATS),
89
+ Param("quality", "int", "JPEG/WebP quality.", default=95, minimum=1, maximum=100),
90
+ Param("recursive", "bool", "Include sub-folders.", default=True),
91
+ Param("workers", "int", parallel.WORKERS_HELP, default=0, minimum=0, maximum=32),
92
+ )
93
+
94
+
95
+ @operation(
96
+ "image.autoorient",
97
+ title="Apply EXIF orientation",
98
+ category="image",
99
+ summary="Bake rotation from the EXIF tag into the pixels and strip it, so every reader agrees.",
100
+ params=ORIENT_PARAMS,
101
+ destructive=True,
102
+ notes=(
103
+ "A phone photo is often stored sideways with a tag saying 'rotate me'.\n"
104
+ "Pillow and most annotation GUIs honour that tag; OpenCV and most\n"
105
+ "training loaders ignore it. So the person drawing the boxes sees an\n"
106
+ "upright image, the trainer sees a sideways one, and the labels land in\n"
107
+ "the wrong place -- with nothing anywhere reporting an error.\n"
108
+ "\n"
109
+ "This rewrites the pixels in their displayed orientation, rotates the\n"
110
+ "annotations with them, and leaves no tag behind to disagree about.\n"
111
+ "Run it first, before anything else touches the images."
112
+ ),
113
+ )
114
+ def autoorient(params: dict[str, Any], ctx: Context) -> OpResult:
115
+ result = OpResult(op="image.autoorient", dry_run=ctx.dry_run)
116
+ rotated = 0
117
+
118
+ def build(width: int, height: int):
119
+ return None # decided per file below, since it depends on the EXIF tag
120
+
121
+ dataset, target_format = _load(params, ctx, result)
122
+ if dataset is None:
123
+ return result
124
+
125
+ writer = DatasetWriter(
126
+ params["output"],
127
+ target_format,
128
+ ctx,
129
+ classes=dataset.labels,
130
+ quality=params["quality"],
131
+ )
132
+ total = len(dataset.items)
133
+ for position, item in enumerate(dataset.items, start=1):
134
+ ctx.check_cancelled()
135
+ ctx.emit(position, total, item.stem)
136
+ if item.image is None:
137
+ continue
138
+ orientation = imaging.exif_orientation(item.image)
139
+ if orientation in (0, 1):
140
+ writer.add(
141
+ item.annotation,
142
+ stem=item.stem,
143
+ source=item.image,
144
+ split=item.split,
145
+ suffix=item.image.suffix,
146
+ )
147
+ continue
148
+ try:
149
+ array = imaging.load(item.image)
150
+ except OSError as exc:
151
+ result.warnings.append(f"{item.image.name}: {exc}")
152
+ continue
153
+ transform = imaging.exif_transform(orientation, array.shape[1], array.shape[0])
154
+ writer.add(
155
+ imaging.transform_annotation(item.annotation, transform),
156
+ stem=item.stem,
157
+ image=imaging.apply_exif(array, orientation),
158
+ source=item.image,
159
+ split=item.split,
160
+ suffix=item.image.suffix,
161
+ )
162
+ rotated += 1
163
+
164
+ report = writer.close()
165
+ result.actions.extend(writer.actions)
166
+ result.counts.update(
167
+ {"images": total, "rotated": rotated, "unchanged": total - rotated, "written": report["labels"]}
168
+ )
169
+ result.warnings.extend(dataset.warnings)
170
+ result.warnings.extend(report["warnings"])
171
+ result.data.update({"output": params["output"], "format": target_format})
172
+ if not rotated:
173
+ result.warnings.append("no image carried a non-upright EXIF orientation")
174
+ return result
175
+
176
+
177
+ TILE_PARAMS = (
178
+ Param("folder", "dir", "Dataset to tile.", required=True, must_exist=True),
179
+ Param("output", "dir", "Where the tiles are written.", required=True),
180
+ Param("tile_width", "int", "Tile width in pixels.", default=640, minimum=32),
181
+ Param("tile_height", "int", "Tile height in pixels.", default=640, minimum=32),
182
+ Param(
183
+ "overlap",
184
+ "float",
185
+ "Fraction each tile overlaps its neighbour (0-0.9).",
186
+ default=0.2,
187
+ minimum=0.0,
188
+ maximum=0.9,
189
+ ),
190
+ Param(
191
+ "min_visibility",
192
+ "float",
193
+ "Fraction of an object that must survive the cut for it to be kept.",
194
+ default=0.3,
195
+ minimum=0.0,
196
+ maximum=1.0,
197
+ ),
198
+ Param("keep_empty", "bool", "Keep tiles containing no objects, as hard negatives.", default=False),
199
+ Param("source_format", "choice", "Input format.", default="auto", choices=SOURCE_FORMATS),
200
+ Param("format", "choice", "Output format.", default="same", choices=OUTPUT_FORMATS),
201
+ Param("quality", "int", "JPEG/WebP quality.", default=95, minimum=1, maximum=100),
202
+ Param("recursive", "bool", "Include sub-folders.", default=True),
203
+ Param("workers", "int", parallel.WORKERS_HELP, default=0, minimum=0, maximum=32),
204
+ )
205
+
206
+
207
+ @operation(
208
+ "image.tile",
209
+ title="Slice into tiles",
210
+ category="image",
211
+ summary="Cut large images into an overlapping grid, reassigning annotations to each tile.",
212
+ params=TILE_PARAMS,
213
+ destructive=True,
214
+ notes=(
215
+ "The fix for small objects in big images. A detector resizes a 4000px\n"
216
+ "frame down to 640 before it ever looks at it, so a 40-pixel object\n"
217
+ "becomes 6 pixels and is effectively invisible. Tiling keeps the native\n"
218
+ "resolution by making each tile its own training image.\n"
219
+ "\n"
220
+ "overlap exists because an object sitting on a tile boundary would\n"
221
+ "otherwise be cut in half in every tile it appears in. At 0.2 an object\n"
222
+ "up to a fifth of a tile wide always lands whole somewhere.\n"
223
+ "\n"
224
+ "min_visibility discards the slivers: without it you teach the model\n"
225
+ "that three pixels of a door is a door.\n"
226
+ "\n"
227
+ "Expect the image count to multiply. A 4000x3000 source at 640 with 0.2\n"
228
+ "overlap yields around 40 tiles."
229
+ ),
230
+ )
231
+ def tile(params: dict[str, Any], ctx: Context) -> OpResult:
232
+ result = OpResult(op="image.tile", dry_run=ctx.dry_run)
233
+ dataset, target_format = _load(params, ctx, result)
234
+ if dataset is None:
235
+ return result
236
+
237
+ tile_w, tile_h = params["tile_width"], params["tile_height"]
238
+ overlap = params["overlap"]
239
+ keep_empty = params["keep_empty"]
240
+
241
+ writer = DatasetWriter(
242
+ params["output"], target_format, ctx, classes=dataset.labels, quality=params["quality"]
243
+ )
244
+ total = len(dataset.items)
245
+ tiles_written = 0
246
+ tiles_empty = 0
247
+ objects_kept = 0
248
+
249
+ for position, item in enumerate(dataset.items, start=1):
250
+ ctx.check_cancelled()
251
+ ctx.emit(position, total, item.stem)
252
+ if item.image is None:
253
+ continue
254
+ try:
255
+ array = imaging.load(item.image)
256
+ except OSError as exc:
257
+ result.warnings.append(f"{item.image.name}: {exc}")
258
+ continue
259
+
260
+ height, width = array.shape[:2]
261
+ for index, (x, y, w, h) in enumerate(
262
+ imaging.tile_windows(width, height, tile_w, tile_h, overlap)
263
+ ):
264
+ crop = imaging.Crop(out_size=(w, h), left=x, top=y)
265
+ annotation = imaging.transform_annotation(
266
+ item.annotation, crop, min_visibility=params["min_visibility"]
267
+ )
268
+ if annotation.is_background and not keep_empty:
269
+ tiles_empty += 1
270
+ continue
271
+ writer.add(
272
+ annotation,
273
+ stem=f"{item.stem}_t{index:03d}",
274
+ image=crop.image(array),
275
+ source=item.image,
276
+ split=item.split,
277
+ suffix=item.image.suffix,
278
+ )
279
+ tiles_written += 1
280
+ objects_kept += len(annotation.shapes)
281
+
282
+ report = writer.close()
283
+ result.actions.extend(writer.actions)
284
+ result.counts.update(
285
+ {
286
+ "source_images": total,
287
+ "tiles_written": tiles_written,
288
+ "tiles_skipped_empty": tiles_empty,
289
+ "objects": objects_kept,
290
+ }
291
+ )
292
+ result.warnings.extend(dataset.warnings)
293
+ result.warnings.extend(report["warnings"])
294
+ result.data.update({"output": params["output"], "format": target_format})
295
+ if tiles_written == 0:
296
+ result.warnings.append(
297
+ "no tiles kept -- every tile came out empty. Use --keep-empty, or "
298
+ "check the tile size is smaller than the images."
299
+ )
300
+ return result
301
+
302
+
303
+ # ------------------------------------------------------------------ shared
304
+
305
+ def _load(params: dict[str, Any], ctx: Context, result: OpResult):
306
+ """Read the source dataset and settle the output format."""
307
+ folder: Path = params["folder"]
308
+ output: Path = params["output"]
309
+ if output.resolve() == folder.resolve():
310
+ result.fail("output must differ from folder -- these operations rewrite pixels")
311
+ return None, ""
312
+ try:
313
+ dataset = formats.read_dataset(
314
+ folder, params["source_format"], recursive=params["recursive"], ctx=ctx
315
+ )
316
+ except AnnotationError as exc:
317
+ result.fail(str(exc))
318
+ return None, ""
319
+ if not dataset.items:
320
+ result.fail(f"no images found in {folder}")
321
+ return None, ""
322
+ chosen = params["format"]
323
+ return dataset, (dataset.source_format if chosen == "same" else chosen)
324
+
325
+
326
+ def _apply_geometry(params, ctx, result, build, *, label: str) -> OpResult:
327
+ """Run a per-image geometric transform over a whole dataset.
328
+
329
+ Decoding, transforming and re-encoding all happen on worker threads; only
330
+ the disk write and the writer's bookkeeping stay on the calling thread,
331
+ which keeps the output order -- and therefore COCO image ids and YOLO class
332
+ indices -- independent of which thread finished first.
333
+ """
334
+ dataset, target_format = _load(params, ctx, result)
335
+ if dataset is None:
336
+ return result
337
+
338
+ writer = DatasetWriter(
339
+ params["output"], target_format, ctx, classes=dataset.labels, quality=params["quality"]
340
+ )
341
+ quality = params["quality"]
342
+ total = len(dataset.items)
343
+ changed = 0
344
+ dropped = 0
345
+
346
+ def transform_one(item):
347
+ """Pure: no shared state, so it is safe on a worker thread."""
348
+ if item.image is None:
349
+ return None
350
+ array = imaging.load(item.image)
351
+ height, width = array.shape[:2]
352
+ transform = build(width, height)
353
+ if transform is None:
354
+ return ("passthrough", item.annotation, None)
355
+ moved = imaging.transform_annotation(item.annotation, transform)
356
+ encoded = imaging.encode(transform.image(array), item.image.suffix, quality=quality)
357
+ return ("changed", moved, encoded)
358
+
359
+ def note_failure(item, exc: Exception) -> None:
360
+ result.warnings.append(f"{item.image.name if item.image else item.stem}: {exc}")
361
+
362
+ for item, outcome in parallel.map_bounded(
363
+ dataset.items,
364
+ transform_one,
365
+ ctx=ctx,
366
+ workers=params["workers"],
367
+ label=lambda item: f"{item.stem} -> {label}",
368
+ on_error=note_failure,
369
+ ):
370
+ if outcome is None:
371
+ continue
372
+ kind, annotation, encoded = outcome
373
+ if kind == "passthrough":
374
+ writer.add(
375
+ annotation,
376
+ stem=item.stem,
377
+ source=item.image,
378
+ split=item.split,
379
+ suffix=item.image.suffix,
380
+ )
381
+ continue
382
+
383
+ dropped += max(0, len(item.annotation.shapes) - len(annotation.shapes))
384
+ writer.add(
385
+ annotation,
386
+ stem=item.stem,
387
+ encoded=encoded,
388
+ source=item.image,
389
+ split=item.split,
390
+ suffix=item.image.suffix,
391
+ )
392
+ changed += 1
393
+
394
+ report = writer.close()
395
+ result.actions.extend(writer.actions)
396
+ result.counts.update(
397
+ {"images": total, "resized": changed, "unchanged": total - changed, "objects_dropped": dropped}
398
+ )
399
+ result.warnings.extend(dataset.warnings)
400
+ result.warnings.extend(report["warnings"])
401
+ result.data.update({"output": params["output"], "format": target_format})
402
+ if dropped:
403
+ result.warnings.append(
404
+ f"{dropped} object(s) became smaller than {imaging.MIN_OBJECT_PIXELS}px "
405
+ f"and were dropped"
406
+ )
407
+ return result