easyannotate 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_factory/__init__.py +82 -0
- data_factory/__main__.py +6 -0
- data_factory/annotate/__init__.py +15 -0
- data_factory/annotate/attach.py +398 -0
- data_factory/annotate/demo.py +161 -0
- data_factory/annotate/images.py +443 -0
- data_factory/annotate/labels.py +399 -0
- data_factory/annotate/loop.py +359 -0
- data_factory/annotate/projects.py +207 -0
- data_factory/annotate/review.py +471 -0
- data_factory/annotate/schema.sql +171 -0
- data_factory/annotate/shapes.py +488 -0
- data_factory/annotate/store.py +292 -0
- data_factory/annotate/tools/__init__.py +39 -0
- data_factory/annotate/tools/fake.py +170 -0
- data_factory/annotate/tools/install.py +254 -0
- data_factory/annotate/tools/local.py +389 -0
- data_factory/annotate/tools/postprocess.py +175 -0
- data_factory/annotate/tools/protocol.py +238 -0
- data_factory/annotate/tools/registry.py +196 -0
- data_factory/annotate/tools/results.py +83 -0
- data_factory/annotate/tools/serve.py +106 -0
- data_factory/cli/__init__.py +21 -0
- data_factory/cli/main.py +837 -0
- data_factory/cli/menu.py +173 -0
- data_factory/cli/render.py +158 -0
- data_factory/core/__init__.py +11 -0
- data_factory/core/annotations/__init__.py +5 -0
- data_factory/core/annotations/coco.py +394 -0
- data_factory/core/annotations/csvfmt.py +174 -0
- data_factory/core/annotations/dota.py +98 -0
- data_factory/core/annotations/formats.py +626 -0
- data_factory/core/annotations/labelme.py +153 -0
- data_factory/core/annotations/model.py +334 -0
- data_factory/core/annotations/voc.py +231 -0
- data_factory/core/annotations/yolo.py +283 -0
- data_factory/core/bktree.py +89 -0
- data_factory/core/fs.py +245 -0
- data_factory/core/imaging.py +626 -0
- data_factory/core/ops/__init__.py +44 -0
- data_factory/core/ops/analyze.py +383 -0
- data_factory/core/ops/annotate_export.py +326 -0
- data_factory/core/ops/audit.py +274 -0
- data_factory/core/ops/augment.py +454 -0
- data_factory/core/ops/collect.py +241 -0
- data_factory/core/ops/combine.py +582 -0
- data_factory/core/ops/convert.py +212 -0
- data_factory/core/ops/dataset.py +246 -0
- data_factory/core/ops/dedup.py +235 -0
- data_factory/core/ops/export.py +168 -0
- data_factory/core/ops/image.py +407 -0
- data_factory/core/ops/labels.py +215 -0
- data_factory/core/ops/pairing.py +133 -0
- data_factory/core/ops/rename.py +154 -0
- data_factory/core/ops/report.py +323 -0
- data_factory/core/ops/visualize.py +164 -0
- data_factory/core/pairs.py +144 -0
- data_factory/core/parallel.py +191 -0
- data_factory/core/provenance.py +136 -0
- data_factory/core/registry.py +339 -0
- data_factory/core/stats.py +403 -0
- data_factory/core/types.py +148 -0
- data_factory/core/writer.py +194 -0
- data_factory/desktop.py +207 -0
- data_factory/hardware.py +243 -0
- data_factory/pipeline.py +192 -0
- data_factory/py.typed +0 -0
- data_factory/service/__init__.py +26 -0
- data_factory/service/app.py +483 -0
- data_factory/service/binding.py +174 -0
- data_factory/service/deps.py +133 -0
- data_factory/service/jobs.py +177 -0
- data_factory/service/limits.py +71 -0
- data_factory/service/routers/__init__.py +11 -0
- data_factory/service/routers/annotate.py +582 -0
- data_factory/service/routers/local.py +464 -0
- data_factory/service/routers/media.py +154 -0
- data_factory/service/routers/predictions.py +116 -0
- data_factory/service/routers/promote.py +182 -0
- data_factory/service/routers/tools.py +308 -0
- data_factory/service/static/annotate/api.js +246 -0
- data_factory/service/static/annotate/app.js +1696 -0
- data_factory/service/static/annotate/assist.js +106 -0
- data_factory/service/static/annotate/filmstrip.js +286 -0
- data_factory/service/static/annotate/hit.js +146 -0
- data_factory/service/static/annotate/labels.js +197 -0
- data_factory/service/static/annotate/render.js +286 -0
- data_factory/service/static/annotate/scene.js +277 -0
- data_factory/service/static/annotate/shapelist.js +72 -0
- data_factory/service/static/annotate/tools/box.js +96 -0
- data_factory/service/static/annotate/tools/line.js +96 -0
- data_factory/service/static/annotate/tools/obb.js +88 -0
- data_factory/service/static/annotate/tools/point.js +49 -0
- data_factory/service/static/annotate/tools/polygon.js +110 -0
- data_factory/service/static/annotate/tools/select.js +227 -0
- data_factory/service/static/annotate/undo.js +176 -0
- data_factory/service/static/annotate/viewport.js +156 -0
- data_factory/service/static/annotate.html +390 -0
- data_factory/service/static/index.html +632 -0
- data_factory/service/workspace.py +244 -0
- easyannotate/__init__.py +64 -0
- easyannotate/__main__.py +5 -0
- easyannotate/project.py +181 -0
- easyannotate/py.typed +0 -0
- easyannotate-0.4.0.dist-info/METADATA +767 -0
- easyannotate-0.4.0.dist-info/RECORD +109 -0
- easyannotate-0.4.0.dist-info/WHEEL +4 -0
- easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
- easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
"""Dataset statistics, and the judgements drawn from them.
|
|
2
|
+
|
|
3
|
+
Counting objects is easy and not very useful. What decides whether a dataset
|
|
4
|
+
trains well is the *shape* of those counts: whether one class has forty times
|
|
5
|
+
another's examples, whether the objects are so small they vanish at the model's
|
|
6
|
+
input resolution, whether half the images are a different size from the rest.
|
|
7
|
+
|
|
8
|
+
Everything here is pure measurement plus explicitly-stated thresholds. The
|
|
9
|
+
thresholds are the arguable part, so they are named constants with a reason
|
|
10
|
+
attached rather than magic numbers buried in a conditional.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import math
|
|
16
|
+
from collections import Counter
|
|
17
|
+
from collections.abc import Iterable
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from .annotations.formats import Dataset
|
|
22
|
+
|
|
23
|
+
#: COCO's own small/medium/large split, in pixels of box area. Using the same
|
|
24
|
+
#: boundaries means our "small object" agrees with everyone's evaluation metric.
|
|
25
|
+
SMALL_AREA = 32 * 32
|
|
26
|
+
MEDIUM_AREA = 96 * 96
|
|
27
|
+
|
|
28
|
+
#: A box under this fraction of the frame survives a 640px resize as roughly a
|
|
29
|
+
#: single pixel, which no detector can learn.
|
|
30
|
+
TINY_FRACTION = 0.0005
|
|
31
|
+
|
|
32
|
+
#: Above this ratio between the commonest and rarest class, the rare class is
|
|
33
|
+
#: usually learned badly no matter how good the model is.
|
|
34
|
+
IMBALANCE_WARN = 10.0
|
|
35
|
+
IMBALANCE_SEVERE = 50.0
|
|
36
|
+
|
|
37
|
+
#: Fewer examples than this and a class is not really trained, it is memorised.
|
|
38
|
+
MIN_EXAMPLES_PER_CLASS = 50
|
|
39
|
+
|
|
40
|
+
#: Hard negatives are valuable, but past this share the model mostly learns to
|
|
41
|
+
#: predict nothing.
|
|
42
|
+
BACKGROUND_WARN = 0.5
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass
|
|
46
|
+
class ClassStat:
|
|
47
|
+
name: str
|
|
48
|
+
objects: int = 0
|
|
49
|
+
images: int = 0
|
|
50
|
+
polygons: int = 0
|
|
51
|
+
areas: list[float] = field(default_factory=list)
|
|
52
|
+
|
|
53
|
+
def as_dict(self, total_objects: int) -> dict[str, Any]:
|
|
54
|
+
return {
|
|
55
|
+
"class": self.name,
|
|
56
|
+
"objects": self.objects,
|
|
57
|
+
"images": self.images,
|
|
58
|
+
"polygons": self.polygons,
|
|
59
|
+
"share": round(self.objects / total_objects, 4) if total_objects else 0.0,
|
|
60
|
+
"median_area_px": int(_median(self.areas)) if self.areas else 0,
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass
|
|
65
|
+
class Finding:
|
|
66
|
+
"""One judgement about the dataset, with what to do about it."""
|
|
67
|
+
|
|
68
|
+
severity: str # "error" | "warning" | "note"
|
|
69
|
+
issue: str
|
|
70
|
+
detail: str
|
|
71
|
+
fix: str
|
|
72
|
+
|
|
73
|
+
def as_dict(self) -> dict[str, str]:
|
|
74
|
+
return {
|
|
75
|
+
"severity": self.severity,
|
|
76
|
+
"issue": self.issue,
|
|
77
|
+
"detail": self.detail,
|
|
78
|
+
"fix": self.fix,
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
_SEVERITY_ORDER = {"error": 0, "warning": 1, "note": 2}
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def analyse(dataset: Dataset) -> dict[str, Any]:
|
|
86
|
+
"""Measure a loaded dataset and return the full statistics payload."""
|
|
87
|
+
per_class: dict[str, ClassStat] = {}
|
|
88
|
+
image_sizes: Counter[tuple[int, int]] = Counter()
|
|
89
|
+
objects_per_image: Counter[int] = Counter()
|
|
90
|
+
relative_areas: list[float] = []
|
|
91
|
+
aspect_ratios: list[float] = []
|
|
92
|
+
size_buckets = Counter({"tiny": 0, "small": 0, "medium": 0, "large": 0})
|
|
93
|
+
per_split: Counter[str] = Counter()
|
|
94
|
+
|
|
95
|
+
total_objects = 0
|
|
96
|
+
backgrounds = 0
|
|
97
|
+
polygon_objects = 0
|
|
98
|
+
out_of_bounds = 0
|
|
99
|
+
degenerate = 0
|
|
100
|
+
|
|
101
|
+
for item in dataset.items:
|
|
102
|
+
annotation = item.annotation
|
|
103
|
+
per_split[item.split or "(none)"] += 1
|
|
104
|
+
if annotation.width > 0 and annotation.height > 0:
|
|
105
|
+
image_sizes[(annotation.width, annotation.height)] += 1
|
|
106
|
+
objects_per_image[len(annotation.shapes)] += 1
|
|
107
|
+
if annotation.is_background:
|
|
108
|
+
backgrounds += 1
|
|
109
|
+
|
|
110
|
+
frame_area = float(max(1, annotation.width * annotation.height))
|
|
111
|
+
for shape in annotation.shapes:
|
|
112
|
+
total_objects += 1
|
|
113
|
+
stat = per_class.setdefault(shape.label, ClassStat(shape.label))
|
|
114
|
+
stat.objects += 1
|
|
115
|
+
area = float(shape.bbox_area)
|
|
116
|
+
stat.areas.append(area)
|
|
117
|
+
if shape.has_polygon:
|
|
118
|
+
stat.polygons += 1
|
|
119
|
+
polygon_objects += 1
|
|
120
|
+
|
|
121
|
+
relative_areas.append(area / frame_area)
|
|
122
|
+
if shape.height > 0:
|
|
123
|
+
aspect_ratios.append(shape.width / shape.height)
|
|
124
|
+
if shape.width <= 0 or shape.height <= 0:
|
|
125
|
+
degenerate += 1
|
|
126
|
+
xmin, ymin, xmax, ymax = shape.bbox
|
|
127
|
+
if (
|
|
128
|
+
xmin < 0
|
|
129
|
+
or ymin < 0
|
|
130
|
+
or (annotation.width and xmax > annotation.width)
|
|
131
|
+
or (annotation.height and ymax > annotation.height)
|
|
132
|
+
):
|
|
133
|
+
out_of_bounds += 1
|
|
134
|
+
|
|
135
|
+
if area / frame_area < TINY_FRACTION:
|
|
136
|
+
size_buckets["tiny"] += 1
|
|
137
|
+
elif area < SMALL_AREA:
|
|
138
|
+
size_buckets["small"] += 1
|
|
139
|
+
elif area < MEDIUM_AREA:
|
|
140
|
+
size_buckets["medium"] += 1
|
|
141
|
+
else:
|
|
142
|
+
size_buckets["large"] += 1
|
|
143
|
+
|
|
144
|
+
for label in annotation.label_set:
|
|
145
|
+
per_class[label].images += 1
|
|
146
|
+
|
|
147
|
+
ordered = sorted(per_class.values(), key=lambda s: (-s.objects, s.name))
|
|
148
|
+
payload: dict[str, Any] = {
|
|
149
|
+
"totals": {
|
|
150
|
+
"images": len(dataset.items),
|
|
151
|
+
"objects": total_objects,
|
|
152
|
+
"classes": len(per_class),
|
|
153
|
+
"backgrounds": backgrounds,
|
|
154
|
+
"polygon_objects": polygon_objects,
|
|
155
|
+
"images_with_polygons": sum(1 for i in dataset.items if i.annotation.has_polygons),
|
|
156
|
+
},
|
|
157
|
+
"per_class": [s.as_dict(total_objects) for s in ordered],
|
|
158
|
+
"object_sizes": dict(size_buckets),
|
|
159
|
+
"relative_area": _distribution(relative_areas),
|
|
160
|
+
"aspect_ratio": _distribution(aspect_ratios),
|
|
161
|
+
"objects_per_image": {
|
|
162
|
+
"mean": round(total_objects / len(dataset.items), 2) if dataset.items else 0.0,
|
|
163
|
+
"max": max(objects_per_image) if objects_per_image else 0,
|
|
164
|
+
"histogram": {str(k): v for k, v in sorted(objects_per_image.items())[:20]},
|
|
165
|
+
},
|
|
166
|
+
"image_sizes": {
|
|
167
|
+
"distinct": len(image_sizes),
|
|
168
|
+
"most_common": [
|
|
169
|
+
{"size": f"{w}x{h}", "images": n} for (w, h), n in image_sizes.most_common(5)
|
|
170
|
+
],
|
|
171
|
+
},
|
|
172
|
+
"splits": dict(sorted(per_split.items())),
|
|
173
|
+
"geometry": {"out_of_bounds": out_of_bounds, "degenerate": degenerate},
|
|
174
|
+
"source_format": dataset.source_format,
|
|
175
|
+
}
|
|
176
|
+
payload["findings"] = [f.as_dict() for f in judge(payload, dataset)]
|
|
177
|
+
payload["balance"] = _balance(ordered)
|
|
178
|
+
return payload
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def judge(payload: dict[str, Any], dataset: Dataset) -> list[Finding]:
|
|
182
|
+
"""Turn the measurements into ranked, actionable findings."""
|
|
183
|
+
findings: list[Finding] = []
|
|
184
|
+
totals = payload["totals"]
|
|
185
|
+
classes: list[dict[str, Any]] = payload["per_class"]
|
|
186
|
+
|
|
187
|
+
if totals["objects"] == 0:
|
|
188
|
+
findings.append(
|
|
189
|
+
Finding(
|
|
190
|
+
"error",
|
|
191
|
+
"No objects at all",
|
|
192
|
+
f"{totals['images']} image(s) loaded, none carrying a single annotation.",
|
|
193
|
+
"Check the source format is right -- convert.detect will tell you what "
|
|
194
|
+
"the folder actually is.",
|
|
195
|
+
)
|
|
196
|
+
)
|
|
197
|
+
return findings
|
|
198
|
+
|
|
199
|
+
# -- class balance
|
|
200
|
+
if len(classes) >= 2:
|
|
201
|
+
biggest, smallest = classes[0], classes[-1]
|
|
202
|
+
ratio = biggest["objects"] / max(1, smallest["objects"])
|
|
203
|
+
if ratio >= IMBALANCE_SEVERE:
|
|
204
|
+
findings.append(
|
|
205
|
+
Finding(
|
|
206
|
+
"error",
|
|
207
|
+
"Severe class imbalance",
|
|
208
|
+
f"{biggest['class']} has {biggest['objects']} objects, "
|
|
209
|
+
f"{smallest['class']} has {smallest['objects']} -- a {ratio:.0f}x gap.",
|
|
210
|
+
"Collect more of the rare class, or cap the common one with "
|
|
211
|
+
"dataset.sample --per-class. Class weighting in the trainer helps "
|
|
212
|
+
"but does not substitute for examples.",
|
|
213
|
+
)
|
|
214
|
+
)
|
|
215
|
+
elif ratio >= IMBALANCE_WARN:
|
|
216
|
+
findings.append(
|
|
217
|
+
Finding(
|
|
218
|
+
"warning",
|
|
219
|
+
"Class imbalance",
|
|
220
|
+
f"{biggest['class']} outnumbers {smallest['class']} by {ratio:.0f}x.",
|
|
221
|
+
"Expect weaker recall on the rare class. Report per-class metrics "
|
|
222
|
+
"rather than a single mAP.",
|
|
223
|
+
)
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
thin = [c for c in classes if c["objects"] < MIN_EXAMPLES_PER_CLASS]
|
|
227
|
+
if thin:
|
|
228
|
+
findings.append(
|
|
229
|
+
Finding(
|
|
230
|
+
"warning",
|
|
231
|
+
"Classes with very few examples",
|
|
232
|
+
", ".join(f"{c['class']} ({c['objects']})" for c in thin[:6])
|
|
233
|
+
+ (f" and {len(thin) - 6} more" if len(thin) > 6 else ""),
|
|
234
|
+
f"Under about {MIN_EXAMPLES_PER_CLASS} objects a class is memorised, "
|
|
235
|
+
f"not learned. Collect more, or merge it into a neighbouring class "
|
|
236
|
+
f"with labels.rename.",
|
|
237
|
+
)
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
# -- case-variant classes: the bug that started this whole tool
|
|
241
|
+
folded: dict[str, list[str]] = {}
|
|
242
|
+
for entry in classes:
|
|
243
|
+
folded.setdefault(entry["class"].strip().casefold(), []).append(entry["class"])
|
|
244
|
+
collisions = {k: v for k, v in folded.items() if len(v) > 1}
|
|
245
|
+
if collisions:
|
|
246
|
+
findings.append(
|
|
247
|
+
Finding(
|
|
248
|
+
"error",
|
|
249
|
+
"Classes differing only by capitalisation",
|
|
250
|
+
"; ".join(f"{' / '.join(sorted(v))}" for v in collisions.values()),
|
|
251
|
+
"These train as separate classes and split your data in half. "
|
|
252
|
+
"Fix with labels.rename --fold-case.",
|
|
253
|
+
)
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
# -- object size
|
|
257
|
+
sizes = payload["object_sizes"]
|
|
258
|
+
tiny_share = sizes["tiny"] / totals["objects"]
|
|
259
|
+
if tiny_share > 0.2:
|
|
260
|
+
findings.append(
|
|
261
|
+
Finding(
|
|
262
|
+
"warning",
|
|
263
|
+
"Many objects are tiny relative to the frame",
|
|
264
|
+
f"{sizes['tiny']} of {totals['objects']} objects ({tiny_share:.0%}) cover "
|
|
265
|
+
f"under {TINY_FRACTION:.2%} of their image.",
|
|
266
|
+
"At a 640px input these are a pixel or two across. Use image.tile to "
|
|
267
|
+
"keep native resolution, or train at a higher input size.",
|
|
268
|
+
)
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
# -- geometry
|
|
272
|
+
geometry = payload["geometry"]
|
|
273
|
+
if geometry["degenerate"]:
|
|
274
|
+
findings.append(
|
|
275
|
+
Finding(
|
|
276
|
+
"error",
|
|
277
|
+
"Zero-area boxes",
|
|
278
|
+
f"{geometry['degenerate']} object(s) have no width or no height.",
|
|
279
|
+
"These crash some trainers and are ignored by others. Delete them, or "
|
|
280
|
+
"re-annotate those images.",
|
|
281
|
+
)
|
|
282
|
+
)
|
|
283
|
+
if geometry["out_of_bounds"]:
|
|
284
|
+
findings.append(
|
|
285
|
+
Finding(
|
|
286
|
+
"warning",
|
|
287
|
+
"Boxes outside the image bounds",
|
|
288
|
+
f"{geometry['out_of_bounds']} object(s) extend past the image edge.",
|
|
289
|
+
"Usually harmless -- most trainers clip -- but it often means the "
|
|
290
|
+
"annotations were made against a different image size. Worth a look.",
|
|
291
|
+
)
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
# -- backgrounds
|
|
295
|
+
background_share = totals["backgrounds"] / max(1, totals["images"])
|
|
296
|
+
if background_share > BACKGROUND_WARN:
|
|
297
|
+
findings.append(
|
|
298
|
+
Finding(
|
|
299
|
+
"warning",
|
|
300
|
+
"Mostly background images",
|
|
301
|
+
f"{totals['backgrounds']} of {totals['images']} images ({background_share:.0%}) "
|
|
302
|
+
f"contain no objects.",
|
|
303
|
+
"Some hard negatives are good (5-15% is typical). This many pushes the "
|
|
304
|
+
"model towards predicting nothing.",
|
|
305
|
+
)
|
|
306
|
+
)
|
|
307
|
+
elif totals["backgrounds"] == 0:
|
|
308
|
+
findings.append(
|
|
309
|
+
Finding(
|
|
310
|
+
"note",
|
|
311
|
+
"No hard negatives",
|
|
312
|
+
"Every image contains at least one object.",
|
|
313
|
+
"Adding some object-free images of the same scenes usually cuts false "
|
|
314
|
+
"positives noticeably.",
|
|
315
|
+
)
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
# -- image sizes
|
|
319
|
+
if payload["image_sizes"]["distinct"] > 1:
|
|
320
|
+
findings.append(
|
|
321
|
+
Finding(
|
|
322
|
+
"note",
|
|
323
|
+
"Mixed image sizes",
|
|
324
|
+
f"{payload['image_sizes']['distinct']} distinct resolutions.",
|
|
325
|
+
"Fine for most trainers, which resize anyway. Run image.resize if you "
|
|
326
|
+
"want a uniform set you can inspect.",
|
|
327
|
+
)
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
# -- splits
|
|
331
|
+
splits = {k: v for k, v in payload["splits"].items() if k != "(none)"}
|
|
332
|
+
if not splits:
|
|
333
|
+
findings.append(
|
|
334
|
+
Finding(
|
|
335
|
+
"note",
|
|
336
|
+
"No train/val split",
|
|
337
|
+
"The dataset is one undivided folder.",
|
|
338
|
+
"Split it with dataset.split before training, and check for leakage "
|
|
339
|
+
"afterwards with inspect.leakage.",
|
|
340
|
+
)
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
if dataset.warnings:
|
|
344
|
+
findings.append(
|
|
345
|
+
Finding(
|
|
346
|
+
"warning",
|
|
347
|
+
"Problems while reading the dataset",
|
|
348
|
+
"; ".join(dataset.warnings[:3])
|
|
349
|
+
+ (f" (+{len(dataset.warnings) - 3} more)" if len(dataset.warnings) > 3 else ""),
|
|
350
|
+
"Files that fail to parse are silently absent from training. Fix or "
|
|
351
|
+
"remove them.",
|
|
352
|
+
)
|
|
353
|
+
)
|
|
354
|
+
|
|
355
|
+
findings.sort(key=lambda f: _SEVERITY_ORDER.get(f.severity, 3))
|
|
356
|
+
return findings
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def _balance(classes: Iterable[ClassStat]) -> dict[str, Any]:
|
|
360
|
+
counts = [c.objects for c in classes]
|
|
361
|
+
if not counts:
|
|
362
|
+
return {"ratio": 0.0, "gini": 0.0}
|
|
363
|
+
ratio = max(counts) / max(1, min(counts))
|
|
364
|
+
return {"ratio": round(ratio, 2), "gini": round(_gini(counts), 3)}
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def _gini(values: list[int]) -> float:
|
|
368
|
+
"""0 = perfectly balanced classes, approaching 1 = one class dominates."""
|
|
369
|
+
if not values or sum(values) == 0:
|
|
370
|
+
return 0.0
|
|
371
|
+
ordered = sorted(values)
|
|
372
|
+
n = len(ordered)
|
|
373
|
+
cumulative = sum((i + 1) * v for i, v in enumerate(ordered))
|
|
374
|
+
return (2 * cumulative) / (n * sum(ordered)) - (n + 1) / n
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _distribution(values: list[float]) -> dict[str, float]:
|
|
378
|
+
if not values:
|
|
379
|
+
return {"min": 0.0, "p25": 0.0, "median": 0.0, "p75": 0.0, "max": 0.0, "mean": 0.0}
|
|
380
|
+
ordered = sorted(values)
|
|
381
|
+
return {
|
|
382
|
+
"min": round(ordered[0], 5),
|
|
383
|
+
"p25": round(_quantile(ordered, 0.25), 5),
|
|
384
|
+
"median": round(_quantile(ordered, 0.5), 5),
|
|
385
|
+
"p75": round(_quantile(ordered, 0.75), 5),
|
|
386
|
+
"max": round(ordered[-1], 5),
|
|
387
|
+
"mean": round(sum(ordered) / len(ordered), 5),
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _quantile(ordered: list[float], q: float) -> float:
|
|
392
|
+
if not ordered:
|
|
393
|
+
return 0.0
|
|
394
|
+
position = (len(ordered) - 1) * q
|
|
395
|
+
low = math.floor(position)
|
|
396
|
+
high = math.ceil(position)
|
|
397
|
+
if low == high:
|
|
398
|
+
return ordered[low]
|
|
399
|
+
return ordered[low] * (high - position) + ordered[high] * (position - low)
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def _median(values: list[float]) -> float:
|
|
403
|
+
return _quantile(sorted(values), 0.5) if values else 0.0
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""Core value types shared by every Data Factory operation.
|
|
2
|
+
|
|
3
|
+
Nothing in this module prints, calls ``sys.exit`` or touches ``sys.argv``.
|
|
4
|
+
Operations return an :class:`OpResult`; adapters (CLI, web API, notebook)
|
|
5
|
+
decide how to render it. That separation is what lets the same code back a
|
|
6
|
+
terminal command and an HTTP endpoint.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from enum import Enum
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any, Protocol
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Action(str, Enum):
|
|
19
|
+
"""What an operation did (or would do) to a single file."""
|
|
20
|
+
|
|
21
|
+
MOVE = "move"
|
|
22
|
+
COPY = "copy"
|
|
23
|
+
DELETE = "delete"
|
|
24
|
+
RENAME = "rename"
|
|
25
|
+
WRITE = "write"
|
|
26
|
+
SKIP = "skip"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(slots=True)
|
|
30
|
+
class FileAction:
|
|
31
|
+
"""One file-level change, recorded whether or not it was actually applied.
|
|
32
|
+
|
|
33
|
+
In dry-run mode operations build the exact same list of ``FileAction``s and
|
|
34
|
+
simply never hand them to :func:`data_factory.core.fs.apply`. A preview is
|
|
35
|
+
therefore a faithful description of the real run, not a separate code path.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
action: Action
|
|
39
|
+
src: Path
|
|
40
|
+
dst: Path | None = None
|
|
41
|
+
reason: str = ""
|
|
42
|
+
|
|
43
|
+
def as_dict(self) -> dict[str, Any]:
|
|
44
|
+
return {
|
|
45
|
+
"action": self.action.value,
|
|
46
|
+
"src": str(self.src),
|
|
47
|
+
"dst": str(self.dst) if self.dst else None,
|
|
48
|
+
"reason": self.reason,
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass
|
|
53
|
+
class OpResult:
|
|
54
|
+
"""The structured return value of every operation.
|
|
55
|
+
|
|
56
|
+
``counts`` is the headline summary an adapter shows first, ``actions`` is
|
|
57
|
+
the audit trail, and ``data`` carries op-specific payload (a label
|
|
58
|
+
histogram, a written manifest path, a dataset report) that does not fit a
|
|
59
|
+
plain integer.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
op: str
|
|
63
|
+
ok: bool = True
|
|
64
|
+
dry_run: bool = False
|
|
65
|
+
counts: dict[str, int] = field(default_factory=dict)
|
|
66
|
+
actions: list[FileAction] = field(default_factory=list)
|
|
67
|
+
warnings: list[str] = field(default_factory=list)
|
|
68
|
+
errors: list[str] = field(default_factory=list)
|
|
69
|
+
data: dict[str, Any] = field(default_factory=dict)
|
|
70
|
+
|
|
71
|
+
def bump(self, key: str, by: int = 1) -> None:
|
|
72
|
+
self.counts[key] = self.counts.get(key, 0) + by
|
|
73
|
+
|
|
74
|
+
def record(self, action: Action, src: Path, dst: Path | None = None, reason: str = "") -> FileAction:
|
|
75
|
+
fa = FileAction(action, src, dst, reason)
|
|
76
|
+
self.actions.append(fa)
|
|
77
|
+
return fa
|
|
78
|
+
|
|
79
|
+
def fail(self, message: str) -> None:
|
|
80
|
+
self.ok = False
|
|
81
|
+
self.errors.append(message)
|
|
82
|
+
|
|
83
|
+
def as_dict(self, include_actions: bool = True) -> dict[str, Any]:
|
|
84
|
+
"""JSON-serialisable form. This is what a web API returns verbatim."""
|
|
85
|
+
out: dict[str, Any] = {
|
|
86
|
+
"op": self.op,
|
|
87
|
+
"ok": self.ok,
|
|
88
|
+
"dry_run": self.dry_run,
|
|
89
|
+
"counts": dict(self.counts),
|
|
90
|
+
"warnings": list(self.warnings),
|
|
91
|
+
"errors": list(self.errors),
|
|
92
|
+
"data": _jsonable(self.data),
|
|
93
|
+
}
|
|
94
|
+
if include_actions:
|
|
95
|
+
out["actions"] = [a.as_dict() for a in self.actions]
|
|
96
|
+
return out
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _jsonable(value: Any) -> Any:
|
|
100
|
+
"""Recursively convert Paths (and containers of them) to strings."""
|
|
101
|
+
if isinstance(value, Path):
|
|
102
|
+
return str(value)
|
|
103
|
+
if isinstance(value, dict):
|
|
104
|
+
return {str(k): _jsonable(v) for k, v in value.items()}
|
|
105
|
+
if isinstance(value, (list, tuple, set)):
|
|
106
|
+
return [_jsonable(v) for v in value]
|
|
107
|
+
return value
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class Progress(Protocol):
|
|
111
|
+
"""Callback shape for progress reporting.
|
|
112
|
+
|
|
113
|
+
The CLI renders a bar, a web app pushes the numbers over a websocket, a
|
|
114
|
+
test passes nothing at all.
|
|
115
|
+
"""
|
|
116
|
+
|
|
117
|
+
def __call__(self, done: int, total: int, message: str = "") -> None: ...
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class Cancelled(Exception):
|
|
121
|
+
"""Raised inside an operation when the caller asked it to stop."""
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@dataclass
|
|
125
|
+
class Context:
|
|
126
|
+
"""Ambient execution settings threaded through every operation.
|
|
127
|
+
|
|
128
|
+
``cancel_check`` exists for the web case: a long dedup over 20k images has
|
|
129
|
+
to be abortable from a browser, and polling a callable is the simplest way
|
|
130
|
+
that works identically in-process and across a task queue.
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
dry_run: bool = False
|
|
134
|
+
progress: Progress | None = None
|
|
135
|
+
log: Callable[[str], None] | None = None
|
|
136
|
+
cancel_check: Callable[[], bool] | None = None
|
|
137
|
+
|
|
138
|
+
def emit(self, done: int, total: int, message: str = "") -> None:
|
|
139
|
+
if self.progress is not None:
|
|
140
|
+
self.progress(done, total, message)
|
|
141
|
+
|
|
142
|
+
def note(self, message: str) -> None:
|
|
143
|
+
if self.log is not None:
|
|
144
|
+
self.log(message)
|
|
145
|
+
|
|
146
|
+
def check_cancelled(self) -> None:
|
|
147
|
+
if self.cancel_check is not None and self.cancel_check():
|
|
148
|
+
raise Cancelled("operation cancelled by caller")
|