easyannotate 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_factory/__init__.py +82 -0
- data_factory/__main__.py +6 -0
- data_factory/annotate/__init__.py +15 -0
- data_factory/annotate/attach.py +398 -0
- data_factory/annotate/demo.py +161 -0
- data_factory/annotate/images.py +443 -0
- data_factory/annotate/labels.py +399 -0
- data_factory/annotate/loop.py +359 -0
- data_factory/annotate/projects.py +207 -0
- data_factory/annotate/review.py +471 -0
- data_factory/annotate/schema.sql +171 -0
- data_factory/annotate/shapes.py +488 -0
- data_factory/annotate/store.py +292 -0
- data_factory/annotate/tools/__init__.py +39 -0
- data_factory/annotate/tools/fake.py +170 -0
- data_factory/annotate/tools/install.py +254 -0
- data_factory/annotate/tools/local.py +389 -0
- data_factory/annotate/tools/postprocess.py +175 -0
- data_factory/annotate/tools/protocol.py +238 -0
- data_factory/annotate/tools/registry.py +196 -0
- data_factory/annotate/tools/results.py +83 -0
- data_factory/annotate/tools/serve.py +106 -0
- data_factory/cli/__init__.py +21 -0
- data_factory/cli/main.py +837 -0
- data_factory/cli/menu.py +173 -0
- data_factory/cli/render.py +158 -0
- data_factory/core/__init__.py +11 -0
- data_factory/core/annotations/__init__.py +5 -0
- data_factory/core/annotations/coco.py +394 -0
- data_factory/core/annotations/csvfmt.py +174 -0
- data_factory/core/annotations/dota.py +98 -0
- data_factory/core/annotations/formats.py +626 -0
- data_factory/core/annotations/labelme.py +153 -0
- data_factory/core/annotations/model.py +334 -0
- data_factory/core/annotations/voc.py +231 -0
- data_factory/core/annotations/yolo.py +283 -0
- data_factory/core/bktree.py +89 -0
- data_factory/core/fs.py +245 -0
- data_factory/core/imaging.py +626 -0
- data_factory/core/ops/__init__.py +44 -0
- data_factory/core/ops/analyze.py +383 -0
- data_factory/core/ops/annotate_export.py +326 -0
- data_factory/core/ops/audit.py +274 -0
- data_factory/core/ops/augment.py +454 -0
- data_factory/core/ops/collect.py +241 -0
- data_factory/core/ops/combine.py +582 -0
- data_factory/core/ops/convert.py +212 -0
- data_factory/core/ops/dataset.py +246 -0
- data_factory/core/ops/dedup.py +235 -0
- data_factory/core/ops/export.py +168 -0
- data_factory/core/ops/image.py +407 -0
- data_factory/core/ops/labels.py +215 -0
- data_factory/core/ops/pairing.py +133 -0
- data_factory/core/ops/rename.py +154 -0
- data_factory/core/ops/report.py +323 -0
- data_factory/core/ops/visualize.py +164 -0
- data_factory/core/pairs.py +144 -0
- data_factory/core/parallel.py +191 -0
- data_factory/core/provenance.py +136 -0
- data_factory/core/registry.py +339 -0
- data_factory/core/stats.py +403 -0
- data_factory/core/types.py +148 -0
- data_factory/core/writer.py +194 -0
- data_factory/desktop.py +207 -0
- data_factory/hardware.py +243 -0
- data_factory/pipeline.py +192 -0
- data_factory/py.typed +0 -0
- data_factory/service/__init__.py +26 -0
- data_factory/service/app.py +483 -0
- data_factory/service/binding.py +174 -0
- data_factory/service/deps.py +133 -0
- data_factory/service/jobs.py +177 -0
- data_factory/service/limits.py +71 -0
- data_factory/service/routers/__init__.py +11 -0
- data_factory/service/routers/annotate.py +582 -0
- data_factory/service/routers/local.py +464 -0
- data_factory/service/routers/media.py +154 -0
- data_factory/service/routers/predictions.py +116 -0
- data_factory/service/routers/promote.py +182 -0
- data_factory/service/routers/tools.py +308 -0
- data_factory/service/static/annotate/api.js +246 -0
- data_factory/service/static/annotate/app.js +1696 -0
- data_factory/service/static/annotate/assist.js +106 -0
- data_factory/service/static/annotate/filmstrip.js +286 -0
- data_factory/service/static/annotate/hit.js +146 -0
- data_factory/service/static/annotate/labels.js +197 -0
- data_factory/service/static/annotate/render.js +286 -0
- data_factory/service/static/annotate/scene.js +277 -0
- data_factory/service/static/annotate/shapelist.js +72 -0
- data_factory/service/static/annotate/tools/box.js +96 -0
- data_factory/service/static/annotate/tools/line.js +96 -0
- data_factory/service/static/annotate/tools/obb.js +88 -0
- data_factory/service/static/annotate/tools/point.js +49 -0
- data_factory/service/static/annotate/tools/polygon.js +110 -0
- data_factory/service/static/annotate/tools/select.js +227 -0
- data_factory/service/static/annotate/undo.js +176 -0
- data_factory/service/static/annotate/viewport.js +156 -0
- data_factory/service/static/annotate.html +390 -0
- data_factory/service/static/index.html +632 -0
- data_factory/service/workspace.py +244 -0
- easyannotate/__init__.py +64 -0
- easyannotate/__main__.py +5 -0
- easyannotate/project.py +181 -0
- easyannotate/py.typed +0 -0
- easyannotate-0.4.0.dist-info/METADATA +767 -0
- easyannotate-0.4.0.dist-info/RECORD +109 -0
- easyannotate-0.4.0.dist-info/WHEEL +4 -0
- easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
- easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,626 @@
|
|
|
1
|
+
"""One dataset, five formats, read and written through a single interface.
|
|
2
|
+
|
|
3
|
+
This is the module that turns a pile of readers and writers into a converter.
|
|
4
|
+
Because everything meets in the middle at :class:`Annotation`, supporting N
|
|
5
|
+
formats costs N readers and N writers -- not the N-squared converters a
|
|
6
|
+
pairwise design would need. Adding CreateML tomorrow means one file, and it
|
|
7
|
+
immediately converts to and from all four existing formats.
|
|
8
|
+
|
|
9
|
+
The layouts recognised are the ones exporters actually produce, not the ones
|
|
10
|
+
the specifications describe:
|
|
11
|
+
|
|
12
|
+
voc ``x.jpg`` + ``x.xml`` side by side, or Annotations/ + JPEGImages/
|
|
13
|
+
yolo ``images/`` + ``labels/`` + ``data.yaml``, or flat .txt sidecars
|
|
14
|
+
coco ``_annotations.coco.json``, or ``annotations/instances_*.json``
|
|
15
|
+
labelme ``x.jpg`` + ``x.json`` side by side
|
|
16
|
+
csv a single ``_annotations.csv``
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from collections.abc import Iterable
|
|
22
|
+
from dataclasses import dataclass, field
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
from .. import fs, parallel
|
|
26
|
+
from ..pairs import PairIndex
|
|
27
|
+
from ..types import Action, Context
|
|
28
|
+
from . import coco as coco_fmt
|
|
29
|
+
from . import csvfmt, labelme, voc, yolo
|
|
30
|
+
from . import dota
|
|
31
|
+
from .model import Annotation, AnnotationError
|
|
32
|
+
|
|
33
|
+
FORMATS = ("voc", "yolo", "coco", "labelme", "csv", "dota")
|
|
34
|
+
|
|
35
|
+
#: Formats able to carry a polygon outline. Converting out of one of these into
|
|
36
|
+
#: a box-only format is lossy, and the converter says so rather than quietly
|
|
37
|
+
#: flattening the masks.
|
|
38
|
+
SEGMENTATION_FORMATS = frozenset({"voc", "yolo", "coco", "labelme"})
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(slots=True)
|
|
42
|
+
class Item:
|
|
43
|
+
"""One image and what is known about it."""
|
|
44
|
+
|
|
45
|
+
annotation: Annotation
|
|
46
|
+
image: Path | None = None
|
|
47
|
+
split: str = ""
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def stem(self) -> str:
|
|
51
|
+
source = self.image.stem if self.image else Path(self.annotation.filename).stem
|
|
52
|
+
return source
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass
|
|
56
|
+
class Dataset:
|
|
57
|
+
"""A whole dataset, loaded into the neutral model."""
|
|
58
|
+
|
|
59
|
+
items: list[Item] = field(default_factory=list)
|
|
60
|
+
source_format: str = ""
|
|
61
|
+
classes: list[str] = field(default_factory=list)
|
|
62
|
+
warnings: list[str] = field(default_factory=list)
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def labels(self) -> list[str]:
|
|
66
|
+
if self.classes:
|
|
67
|
+
return self.classes
|
|
68
|
+
found: set[str] = set()
|
|
69
|
+
for item in self.items:
|
|
70
|
+
found |= item.annotation.label_set
|
|
71
|
+
return sorted(found)
|
|
72
|
+
|
|
73
|
+
def stats(self) -> dict[str, int]:
|
|
74
|
+
objects = sum(len(i.annotation.shapes) for i in self.items)
|
|
75
|
+
return {
|
|
76
|
+
"images": len(self.items),
|
|
77
|
+
"objects": objects,
|
|
78
|
+
"classes": len(self.labels),
|
|
79
|
+
"backgrounds": sum(1 for i in self.items if i.annotation.is_background),
|
|
80
|
+
"with_polygons": sum(1 for i in self.items if i.annotation.has_polygons),
|
|
81
|
+
"missing_images": sum(1 for i in self.items if i.image is None),
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
# ------------------------------------------------------------- detection
|
|
86
|
+
|
|
87
|
+
def detect(folder: Path) -> str:
|
|
88
|
+
"""Guess the format of a folder by looking for each layout's marker.
|
|
89
|
+
|
|
90
|
+
Ordered by how distinctive the evidence is: a ``data.yaml`` or a COCO json
|
|
91
|
+
is conclusive, while loose ``.json`` files need their contents inspected to
|
|
92
|
+
tell LabelMe from COCO.
|
|
93
|
+
"""
|
|
94
|
+
folder = Path(folder)
|
|
95
|
+
if yolo.find_data_yaml(folder):
|
|
96
|
+
return "yolo"
|
|
97
|
+
if (folder / yolo.LABEL_DIR).is_dir():
|
|
98
|
+
# DOTA uses the same images/ + labels/ layout; only the lines differ.
|
|
99
|
+
return "dota" if _looks_like_dota(folder) else "yolo"
|
|
100
|
+
|
|
101
|
+
for path in coco_fmt.find_annotation_files(folder):
|
|
102
|
+
if _looks_like_coco(path):
|
|
103
|
+
return "coco"
|
|
104
|
+
|
|
105
|
+
counts = {
|
|
106
|
+
"voc": _count(folder, ".xml"),
|
|
107
|
+
"labelme": _count(folder, ".json"),
|
|
108
|
+
"yolo": _count(folder, ".txt"),
|
|
109
|
+
"csv": _count(folder, ".csv"),
|
|
110
|
+
}
|
|
111
|
+
if counts["voc"]:
|
|
112
|
+
return "voc"
|
|
113
|
+
if counts["labelme"]:
|
|
114
|
+
return "labelme"
|
|
115
|
+
if counts["csv"]:
|
|
116
|
+
return "csv"
|
|
117
|
+
if counts["yolo"]:
|
|
118
|
+
# Both are bare .txt files; DOTA names its class in the ninth field,
|
|
119
|
+
# where YOLO has nothing but numbers.
|
|
120
|
+
return "dota" if _looks_like_dota(folder) else "yolo"
|
|
121
|
+
raise AnnotationError(
|
|
122
|
+
f"cannot tell what format {folder} is -- found no .xml, .txt, .json or "
|
|
123
|
+
f".csv labels. Pass the format explicitly."
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _looks_like_dota(folder: Path) -> bool:
|
|
128
|
+
for path in folder.rglob("*.txt"):
|
|
129
|
+
if not path.is_file() or path.name == "classes.txt":
|
|
130
|
+
continue
|
|
131
|
+
try:
|
|
132
|
+
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
|
133
|
+
except OSError:
|
|
134
|
+
continue
|
|
135
|
+
for line in lines:
|
|
136
|
+
parts = line.split()
|
|
137
|
+
if not parts or line.startswith(dota.HEADER_PREFIXES):
|
|
138
|
+
continue
|
|
139
|
+
if len(parts) < 9:
|
|
140
|
+
return False
|
|
141
|
+
try:
|
|
142
|
+
float(parts[8])
|
|
143
|
+
except ValueError:
|
|
144
|
+
return True
|
|
145
|
+
return False
|
|
146
|
+
return False
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _count(folder: Path, suffix: str, limit: int = 1) -> int:
|
|
150
|
+
seen = 0
|
|
151
|
+
for path in folder.rglob(f"*{suffix}"):
|
|
152
|
+
if path.is_file() and path.name != "data.yaml":
|
|
153
|
+
seen += 1
|
|
154
|
+
if seen >= limit:
|
|
155
|
+
return seen
|
|
156
|
+
return seen
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _looks_like_coco(path: Path) -> bool:
|
|
160
|
+
"""Peek at the head of the file rather than parsing a 200MB json."""
|
|
161
|
+
try:
|
|
162
|
+
with path.open("r", encoding="utf-8") as handle:
|
|
163
|
+
head = handle.read(4096)
|
|
164
|
+
except OSError:
|
|
165
|
+
return False
|
|
166
|
+
return '"categories"' in head or '"annotations"' in head
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
# ---------------------------------------------------------------- reading
|
|
170
|
+
|
|
171
|
+
def read_dataset(
|
|
172
|
+
folder: Path,
|
|
173
|
+
fmt: str = "auto",
|
|
174
|
+
*,
|
|
175
|
+
recursive: bool = True,
|
|
176
|
+
ctx: Context | None = None,
|
|
177
|
+
) -> Dataset:
|
|
178
|
+
"""Load a folder into the neutral model, whatever format it is in."""
|
|
179
|
+
folder = Path(folder)
|
|
180
|
+
resolved = detect(folder) if fmt in ("", "auto") else fmt
|
|
181
|
+
if resolved not in FORMATS:
|
|
182
|
+
raise AnnotationError(f"unknown format {resolved!r}; known: {', '.join(FORMATS)}")
|
|
183
|
+
|
|
184
|
+
reader = {
|
|
185
|
+
"voc": _read_voc,
|
|
186
|
+
"yolo": _read_yolo,
|
|
187
|
+
"coco": _read_coco,
|
|
188
|
+
"labelme": _read_labelme,
|
|
189
|
+
"csv": _read_csv,
|
|
190
|
+
"dota": _read_dota,
|
|
191
|
+
}[resolved]
|
|
192
|
+
dataset = reader(folder, recursive, ctx or Context())
|
|
193
|
+
dataset.source_format = resolved
|
|
194
|
+
return dataset
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _read_sidecars(
|
|
198
|
+
folder: Path, recursive: bool, ext: str, parse
|
|
199
|
+
) -> Dataset:
|
|
200
|
+
"""Shared body for the two sidecar formats (VOC and LabelMe)."""
|
|
201
|
+
dataset = Dataset()
|
|
202
|
+
index = PairIndex.scan(folder, recursive=recursive, label_ext=ext)
|
|
203
|
+
for pair in index:
|
|
204
|
+
if pair.label is None:
|
|
205
|
+
if pair.image is not None:
|
|
206
|
+
# An unlabelled image is a real dataset member; carry it as a
|
|
207
|
+
# background rather than dropping it on the floor.
|
|
208
|
+
dataset.items.append(
|
|
209
|
+
Item(annotation=_empty_annotation(pair.image), image=pair.image)
|
|
210
|
+
)
|
|
211
|
+
continue
|
|
212
|
+
try:
|
|
213
|
+
annotation = parse(pair.label)
|
|
214
|
+
except AnnotationError as exc:
|
|
215
|
+
dataset.warnings.append(str(exc))
|
|
216
|
+
continue
|
|
217
|
+
if pair.image is not None and (annotation.width <= 0 or annotation.height <= 0):
|
|
218
|
+
annotation.width, annotation.height = _size_of(pair.image, dataset)
|
|
219
|
+
dataset.items.append(Item(annotation=annotation, image=pair.image, split=_split_of(pair.key)))
|
|
220
|
+
return dataset
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _read_voc(folder: Path, recursive: bool, ctx: Context) -> Dataset:
|
|
224
|
+
return _read_sidecars(folder, recursive, ".xml", voc.read)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _read_labelme(folder: Path, recursive: bool, ctx: Context) -> Dataset:
|
|
228
|
+
skipped: list[str] = []
|
|
229
|
+
|
|
230
|
+
def parse(path: Path) -> Annotation:
|
|
231
|
+
return labelme.read(path, skipped_out=skipped)
|
|
232
|
+
|
|
233
|
+
dataset = _read_sidecars(folder, recursive, ".json", parse)
|
|
234
|
+
if skipped:
|
|
235
|
+
dataset.warnings.append(
|
|
236
|
+
f"LabelMe shape types not representable as polygons were skipped: "
|
|
237
|
+
f"{', '.join(sorted(set(skipped)))}"
|
|
238
|
+
)
|
|
239
|
+
return dataset
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _read_yolo(folder: Path, recursive: bool, ctx: Context) -> Dataset:
|
|
243
|
+
dataset = Dataset()
|
|
244
|
+
yaml_path = yolo.find_data_yaml(folder)
|
|
245
|
+
if yaml_path is not None:
|
|
246
|
+
classes, _ = yolo.read_data_yaml(yaml_path)
|
|
247
|
+
dataset.classes = list(classes.names)
|
|
248
|
+
else:
|
|
249
|
+
# Without a class list the indices are meaningless, so say so loudly
|
|
250
|
+
# and fall back to placeholder names rather than inventing a mapping.
|
|
251
|
+
classes = yolo.ClassList([])
|
|
252
|
+
dataset.warnings.append(
|
|
253
|
+
"no data.yaml found -- classes will be named class_0, class_1, ... "
|
|
254
|
+
"Point --classes at the right file to keep the real names."
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
# YOLO stores coordinates normalised, so every label needs its image's
|
|
258
|
+
# dimensions before it can be read at all -- one header read per image.
|
|
259
|
+
# Serial, that is the dominant cost of loading a large YOLO dataset.
|
|
260
|
+
images = list(fs.iter_images(folder, recursive=recursive))
|
|
261
|
+
sizes = dict(
|
|
262
|
+
parallel.collect(
|
|
263
|
+
images,
|
|
264
|
+
_image_size,
|
|
265
|
+
ctx=ctx,
|
|
266
|
+
label=lambda image: image.name,
|
|
267
|
+
on_error=lambda image, exc: dataset.warnings.append(
|
|
268
|
+
f"{image.name}: cannot read image size ({exc})"
|
|
269
|
+
),
|
|
270
|
+
)
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
for image in images:
|
|
274
|
+
label_path = yolo.label_path_for(image, folder)
|
|
275
|
+
width, height = sizes.get(image, (0, 0))
|
|
276
|
+
if width <= 0:
|
|
277
|
+
continue
|
|
278
|
+
if not label_path.is_file():
|
|
279
|
+
dataset.items.append(
|
|
280
|
+
Item(annotation=Annotation(filename=image.name, width=width, height=height),
|
|
281
|
+
image=image)
|
|
282
|
+
)
|
|
283
|
+
continue
|
|
284
|
+
try:
|
|
285
|
+
annotation = yolo.read(
|
|
286
|
+
label_path, classes, width, height, filename=image.name
|
|
287
|
+
)
|
|
288
|
+
except AnnotationError as exc:
|
|
289
|
+
dataset.warnings.append(str(exc))
|
|
290
|
+
continue
|
|
291
|
+
dataset.items.append(
|
|
292
|
+
Item(
|
|
293
|
+
annotation=annotation,
|
|
294
|
+
image=image,
|
|
295
|
+
split=_split_of(str(fs.relative_to_safe(image, folder)).replace("\\", "/")),
|
|
296
|
+
)
|
|
297
|
+
)
|
|
298
|
+
if not dataset.classes:
|
|
299
|
+
dataset.classes = list(classes.names)
|
|
300
|
+
return dataset
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _read_dota(folder: Path, recursive: bool, ctx: Context) -> Dataset:
|
|
304
|
+
"""Like YOLO -- one .txt beside or under labels/ per image -- but the class
|
|
305
|
+
is named in the file, so there is no list to find."""
|
|
306
|
+
dataset = Dataset()
|
|
307
|
+
images = list(fs.iter_images(folder, recursive=recursive))
|
|
308
|
+
sizes = dict(
|
|
309
|
+
parallel.collect(
|
|
310
|
+
images,
|
|
311
|
+
_image_size,
|
|
312
|
+
ctx=ctx,
|
|
313
|
+
label=lambda image: image.name,
|
|
314
|
+
on_error=lambda image, exc: dataset.warnings.append(
|
|
315
|
+
f"{image.name}: cannot read image size ({exc})"
|
|
316
|
+
),
|
|
317
|
+
)
|
|
318
|
+
)
|
|
319
|
+
for image in images:
|
|
320
|
+
label_path = yolo.label_path_for(image, folder)
|
|
321
|
+
width, height = sizes.get(image, (0, 0))
|
|
322
|
+
if width <= 0:
|
|
323
|
+
continue
|
|
324
|
+
if not label_path.is_file():
|
|
325
|
+
dataset.items.append(
|
|
326
|
+
Item(annotation=Annotation(filename=image.name, width=width, height=height),
|
|
327
|
+
image=image)
|
|
328
|
+
)
|
|
329
|
+
continue
|
|
330
|
+
try:
|
|
331
|
+
annotation = dota.read(label_path, width, height, filename=image.name)
|
|
332
|
+
except AnnotationError as exc:
|
|
333
|
+
dataset.warnings.append(str(exc))
|
|
334
|
+
continue
|
|
335
|
+
dataset.items.append(
|
|
336
|
+
Item(
|
|
337
|
+
annotation=annotation,
|
|
338
|
+
image=image,
|
|
339
|
+
split=_split_of(str(fs.relative_to_safe(image, folder)).replace("\\", "/")),
|
|
340
|
+
)
|
|
341
|
+
)
|
|
342
|
+
return dataset
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _read_coco(folder: Path, recursive: bool, ctx: Context) -> Dataset:
|
|
346
|
+
dataset = Dataset()
|
|
347
|
+
files = [p for p in coco_fmt.find_annotation_files(folder) if _looks_like_coco(p)]
|
|
348
|
+
if not files:
|
|
349
|
+
raise AnnotationError(f"no COCO annotation json found under {folder}")
|
|
350
|
+
|
|
351
|
+
images_by_name = _image_index(folder)
|
|
352
|
+
unsupported: list[str] = []
|
|
353
|
+
for path in files:
|
|
354
|
+
document = coco_fmt.load(path)
|
|
355
|
+
split = _split_of(path.stem)
|
|
356
|
+
for annotation in coco_fmt.to_annotations(document, unsupported_out=unsupported):
|
|
357
|
+
image = _pick_image(images_by_name, annotation.filename, split)
|
|
358
|
+
dataset.items.append(Item(annotation=annotation, image=image, split=split))
|
|
359
|
+
for category in document.get("categories", []):
|
|
360
|
+
name = str(category.get("name", ""))
|
|
361
|
+
if name and name not in dataset.classes:
|
|
362
|
+
dataset.classes.append(name)
|
|
363
|
+
dataset.warnings.extend(unsupported)
|
|
364
|
+
return dataset
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def _read_csv(folder: Path, recursive: bool, ctx: Context) -> Dataset:
|
|
368
|
+
dataset = Dataset()
|
|
369
|
+
candidates = sorted(folder.rglob("*.csv")) if recursive else sorted(folder.glob("*.csv"))
|
|
370
|
+
if not candidates:
|
|
371
|
+
raise AnnotationError(f"no .csv annotation file found under {folder}")
|
|
372
|
+
|
|
373
|
+
images_by_name = _image_index(folder)
|
|
374
|
+
for path in candidates:
|
|
375
|
+
for annotation in csvfmt.read(path):
|
|
376
|
+
image = _pick_image(images_by_name, annotation.filename)
|
|
377
|
+
if image is not None and annotation.width <= 0:
|
|
378
|
+
annotation.width, annotation.height = _size_of(image, dataset)
|
|
379
|
+
dataset.items.append(Item(annotation=annotation, image=image))
|
|
380
|
+
return dataset
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _image_index(folder: Path) -> dict[str, list[Path]]:
|
|
384
|
+
"""Every image under a folder, findable by both name and stem.
|
|
385
|
+
|
|
386
|
+
COCO and CSV reference images by name only, so the file could be anywhere
|
|
387
|
+
in the tree -- and in a split dataset ``a.jpg`` legitimately exists in both
|
|
388
|
+
``train/`` and ``val/``. Every candidate is kept so the caller can pick the
|
|
389
|
+
one in the right split rather than whichever sorted first.
|
|
390
|
+
"""
|
|
391
|
+
index: dict[str, list[Path]] = {}
|
|
392
|
+
for image in fs.iter_images(folder, recursive=True):
|
|
393
|
+
index.setdefault(image.name, []).append(image)
|
|
394
|
+
if image.stem != image.name:
|
|
395
|
+
index.setdefault(image.stem, []).append(image)
|
|
396
|
+
return index
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def _pick_image(
|
|
400
|
+
index: dict[str, list[Path]], filename: str, split: str = ""
|
|
401
|
+
) -> Path | None:
|
|
402
|
+
"""Resolve a referenced filename to an actual file, preferring its split.
|
|
403
|
+
|
|
404
|
+
Without the split preference, a COCO dataset whose train and val folders
|
|
405
|
+
share a filename would attach val's annotations to train's image -- wrong
|
|
406
|
+
in a way nothing downstream could detect.
|
|
407
|
+
"""
|
|
408
|
+
candidates = index.get(filename) or index.get(Path(filename).name) or index.get(
|
|
409
|
+
Path(filename).stem
|
|
410
|
+
)
|
|
411
|
+
if not candidates:
|
|
412
|
+
return None
|
|
413
|
+
if split:
|
|
414
|
+
for candidate in candidates:
|
|
415
|
+
if split in {part.lower() for part in candidate.parts}:
|
|
416
|
+
return candidate
|
|
417
|
+
return candidates[0]
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _empty_annotation(image: Path) -> Annotation:
|
|
421
|
+
try:
|
|
422
|
+
width, height = _image_size(image)
|
|
423
|
+
except Exception:
|
|
424
|
+
width, height = 0, 0
|
|
425
|
+
return Annotation(filename=image.name, width=width, height=height, shapes=[])
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def _size_of(image: Path, dataset: Dataset) -> tuple[int, int]:
|
|
429
|
+
try:
|
|
430
|
+
return _image_size(image)
|
|
431
|
+
except Exception as exc:
|
|
432
|
+
dataset.warnings.append(f"{image.name}: cannot read image size ({exc})")
|
|
433
|
+
return 0, 0
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def _image_size(path: Path) -> tuple[int, int]:
|
|
437
|
+
from ..imaging import image_size
|
|
438
|
+
|
|
439
|
+
return image_size(path)
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _split_of(key: str) -> str:
|
|
443
|
+
"""Recover train/val/test from a path or filename, when it is there."""
|
|
444
|
+
lowered = key.lower().replace("\\", "/")
|
|
445
|
+
for name in ("train", "valid", "val", "test"):
|
|
446
|
+
if f"/{name}/" in f"/{lowered}/" or lowered.endswith(f"_{name}"):
|
|
447
|
+
return "val" if name == "valid" else name
|
|
448
|
+
return ""
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
# ---------------------------------------------------------------- writing
|
|
452
|
+
|
|
453
|
+
def write_dataset(
|
|
454
|
+
dataset: Dataset,
|
|
455
|
+
output: Path,
|
|
456
|
+
fmt: str,
|
|
457
|
+
*,
|
|
458
|
+
ctx: Context,
|
|
459
|
+
copy_images: bool = True,
|
|
460
|
+
segmentation: bool = True,
|
|
461
|
+
on_action=None,
|
|
462
|
+
) -> dict[str, object]:
|
|
463
|
+
"""Serialise a dataset into ``fmt`` under ``output``.
|
|
464
|
+
|
|
465
|
+
Image files are copied through the usual :func:`fs.perform` choke point, so
|
|
466
|
+
``--dry-run`` reports the entire conversion without writing a byte. Label
|
|
467
|
+
files are written directly, but only ever inside ``output``, and never at
|
|
468
|
+
all in dry-run mode.
|
|
469
|
+
"""
|
|
470
|
+
if fmt not in FORMATS:
|
|
471
|
+
raise AnnotationError(f"unknown format {fmt!r}; known: {', '.join(FORMATS)}")
|
|
472
|
+
|
|
473
|
+
output = Path(output)
|
|
474
|
+
report: dict[str, object] = {"format": fmt, "written": 0, "images": 0, "warnings": []}
|
|
475
|
+
warnings: list[str] = report["warnings"] # type: ignore[assignment]
|
|
476
|
+
|
|
477
|
+
if segmentation and fmt not in SEGMENTATION_FORMATS:
|
|
478
|
+
polygons = sum(1 for i in dataset.items if i.annotation.has_polygons)
|
|
479
|
+
if polygons:
|
|
480
|
+
warnings.append(
|
|
481
|
+
f"{fmt} cannot store outlines: {polygons} image(s) keep their boxes "
|
|
482
|
+
f"but lose their polygons"
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
writer = {
|
|
486
|
+
"voc": _write_voc,
|
|
487
|
+
"yolo": _write_yolo,
|
|
488
|
+
"coco": _write_coco,
|
|
489
|
+
"labelme": _write_labelme,
|
|
490
|
+
"csv": _write_csv,
|
|
491
|
+
"dota": _write_dota,
|
|
492
|
+
}[fmt]
|
|
493
|
+
writer(dataset, output, ctx, report, copy_images, segmentation, on_action)
|
|
494
|
+
return report
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def _copy_image(
|
|
498
|
+
item: Item, destination: Path, ctx: Context, report: dict, on_action
|
|
499
|
+
) -> None:
|
|
500
|
+
if item.image is None or not destination:
|
|
501
|
+
return
|
|
502
|
+
action = fs.FileAction(Action.COPY, item.image, destination, "convert")
|
|
503
|
+
fs.perform(action, ctx)
|
|
504
|
+
report["images"] = int(report["images"]) + 1
|
|
505
|
+
if on_action is not None:
|
|
506
|
+
on_action(action)
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _split_dir(base: Path, item: Item) -> Path:
|
|
510
|
+
return base / item.split if item.split else base
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def _write_voc(dataset, output, ctx, report, copy_images, segmentation, on_action) -> None:
|
|
514
|
+
for item in dataset.items:
|
|
515
|
+
folder = _split_dir(output, item)
|
|
516
|
+
fs.ensure_dir(folder, ctx)
|
|
517
|
+
stem = item.stem
|
|
518
|
+
if copy_images and item.image is not None:
|
|
519
|
+
_copy_image(item, folder / f"{stem}{item.image.suffix}", ctx, report, on_action)
|
|
520
|
+
if not ctx.dry_run:
|
|
521
|
+
voc.write(item.annotation, folder / f"{stem}.xml")
|
|
522
|
+
report["written"] = int(report["written"]) + 1
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
def _write_labelme(dataset, output, ctx, report, copy_images, segmentation, on_action) -> None:
|
|
526
|
+
for item in dataset.items:
|
|
527
|
+
folder = _split_dir(output, item)
|
|
528
|
+
fs.ensure_dir(folder, ctx)
|
|
529
|
+
stem = item.stem
|
|
530
|
+
if copy_images and item.image is not None:
|
|
531
|
+
_copy_image(item, folder / f"{stem}{item.image.suffix}", ctx, report, on_action)
|
|
532
|
+
if not ctx.dry_run:
|
|
533
|
+
labelme.write(item.annotation, folder / f"{stem}.json")
|
|
534
|
+
report["written"] = int(report["written"]) + 1
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
def _write_yolo(dataset, output, ctx, report, copy_images, segmentation, on_action) -> None:
|
|
538
|
+
# Seed the class list from the dataset so indices follow the declared order
|
|
539
|
+
# rather than whichever image happened to be processed first.
|
|
540
|
+
classes = yolo.ClassList(list(dataset.labels))
|
|
541
|
+
splits: dict[str, str] = {}
|
|
542
|
+
|
|
543
|
+
for item in dataset.items:
|
|
544
|
+
split = item.split or "train"
|
|
545
|
+
splits.setdefault(split, f"{yolo.IMAGE_DIR}/{split}")
|
|
546
|
+
image_dir = output / yolo.IMAGE_DIR / split
|
|
547
|
+
label_dir = output / yolo.LABEL_DIR / split
|
|
548
|
+
fs.ensure_dir(image_dir, ctx)
|
|
549
|
+
fs.ensure_dir(label_dir, ctx)
|
|
550
|
+
stem = item.stem
|
|
551
|
+
if copy_images and item.image is not None:
|
|
552
|
+
_copy_image(item, image_dir / f"{stem}{item.image.suffix}", ctx, report, on_action)
|
|
553
|
+
if not ctx.dry_run:
|
|
554
|
+
yolo.write(item.annotation, label_dir / f"{stem}.txt", classes, segmentation=segmentation)
|
|
555
|
+
report["written"] = int(report["written"]) + 1
|
|
556
|
+
|
|
557
|
+
if not ctx.dry_run:
|
|
558
|
+
yolo.write_data_yaml(output / yolo.DATA_YAML, classes, splits=splits, root=".")
|
|
559
|
+
report["classes"] = list(classes.names)
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def _write_dota(dataset, output, ctx, report, copy_images, segmentation, on_action) -> None:
|
|
563
|
+
for item in dataset.items:
|
|
564
|
+
split = item.split or "train"
|
|
565
|
+
image_dir = output / yolo.IMAGE_DIR / split
|
|
566
|
+
label_dir = output / yolo.LABEL_DIR / split
|
|
567
|
+
fs.ensure_dir(image_dir, ctx)
|
|
568
|
+
fs.ensure_dir(label_dir, ctx)
|
|
569
|
+
stem = item.stem
|
|
570
|
+
if copy_images and item.image is not None:
|
|
571
|
+
_copy_image(item, image_dir / f"{stem}{item.image.suffix}", ctx, report, on_action)
|
|
572
|
+
if not ctx.dry_run:
|
|
573
|
+
dota.write(item.annotation, label_dir / f"{stem}.txt")
|
|
574
|
+
report["written"] = int(report["written"]) + 1
|
|
575
|
+
report["classes"] = list(dataset.labels)
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def _write_coco(dataset, output, ctx, report, copy_images, segmentation, on_action) -> None:
|
|
579
|
+
by_split: dict[str, list[Item]] = {}
|
|
580
|
+
for item in dataset.items:
|
|
581
|
+
by_split.setdefault(item.split or "train", []).append(item)
|
|
582
|
+
|
|
583
|
+
for split, items in sorted(by_split.items()):
|
|
584
|
+
builder = coco_fmt.CocoBuilder(description=f"Data Factory export ({split})")
|
|
585
|
+
image_dir = output / split
|
|
586
|
+
fs.ensure_dir(image_dir, ctx)
|
|
587
|
+
for item in items:
|
|
588
|
+
name = f"{item.stem}{item.image.suffix if item.image else '.jpg'}"
|
|
589
|
+
builder.add(item.annotation, file_name=name)
|
|
590
|
+
if copy_images and item.image is not None:
|
|
591
|
+
_copy_image(item, image_dir / name, ctx, report, on_action)
|
|
592
|
+
report["written"] = int(report["written"]) + 1
|
|
593
|
+
if not ctx.dry_run:
|
|
594
|
+
fs.ensure_dir(output / "annotations", ctx)
|
|
595
|
+
builder.write(output / "annotations" / f"instances_{split}.json")
|
|
596
|
+
merges = builder.categories.merges()
|
|
597
|
+
if merges:
|
|
598
|
+
report["warnings"].append( # type: ignore[union-attr]
|
|
599
|
+
"case-variant classes merged: "
|
|
600
|
+
+ "; ".join(f"{k} <- {', '.join(v)}" for k, v in merges.items())
|
|
601
|
+
)
|
|
602
|
+
|
|
603
|
+
|
|
604
|
+
def _write_csv(dataset, output, ctx, report, copy_images, segmentation, on_action) -> None:
|
|
605
|
+
fs.ensure_dir(output, ctx)
|
|
606
|
+
image_dir = output / "images"
|
|
607
|
+
annotations: list[Annotation] = []
|
|
608
|
+
for item in dataset.items:
|
|
609
|
+
annotations.append(item.annotation)
|
|
610
|
+
if copy_images and item.image is not None:
|
|
611
|
+
fs.ensure_dir(image_dir, ctx)
|
|
612
|
+
_copy_image(
|
|
613
|
+
item, image_dir / f"{item.stem}{item.image.suffix}", ctx, report, on_action
|
|
614
|
+
)
|
|
615
|
+
report["written"] = int(report["written"]) + 1
|
|
616
|
+
if not ctx.dry_run:
|
|
617
|
+
csvfmt.write(annotations, output / "_annotations.csv")
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def label_extension(fmt: str) -> str:
|
|
621
|
+
"""The sidecar extension a format uses, or empty for the collected ones."""
|
|
622
|
+
return {"voc": ".xml", "labelme": ".json", "yolo": ".txt", "dota": ".txt"}.get(fmt, "")
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
def iter_formats() -> Iterable[str]:
|
|
626
|
+
return FORMATS
|