easyannotate 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. data_factory/__init__.py +82 -0
  2. data_factory/__main__.py +6 -0
  3. data_factory/annotate/__init__.py +15 -0
  4. data_factory/annotate/attach.py +398 -0
  5. data_factory/annotate/demo.py +161 -0
  6. data_factory/annotate/images.py +443 -0
  7. data_factory/annotate/labels.py +399 -0
  8. data_factory/annotate/loop.py +359 -0
  9. data_factory/annotate/projects.py +207 -0
  10. data_factory/annotate/review.py +471 -0
  11. data_factory/annotate/schema.sql +171 -0
  12. data_factory/annotate/shapes.py +488 -0
  13. data_factory/annotate/store.py +292 -0
  14. data_factory/annotate/tools/__init__.py +39 -0
  15. data_factory/annotate/tools/fake.py +170 -0
  16. data_factory/annotate/tools/install.py +254 -0
  17. data_factory/annotate/tools/local.py +389 -0
  18. data_factory/annotate/tools/postprocess.py +175 -0
  19. data_factory/annotate/tools/protocol.py +238 -0
  20. data_factory/annotate/tools/registry.py +196 -0
  21. data_factory/annotate/tools/results.py +83 -0
  22. data_factory/annotate/tools/serve.py +106 -0
  23. data_factory/cli/__init__.py +21 -0
  24. data_factory/cli/main.py +837 -0
  25. data_factory/cli/menu.py +173 -0
  26. data_factory/cli/render.py +158 -0
  27. data_factory/core/__init__.py +11 -0
  28. data_factory/core/annotations/__init__.py +5 -0
  29. data_factory/core/annotations/coco.py +394 -0
  30. data_factory/core/annotations/csvfmt.py +174 -0
  31. data_factory/core/annotations/dota.py +98 -0
  32. data_factory/core/annotations/formats.py +626 -0
  33. data_factory/core/annotations/labelme.py +153 -0
  34. data_factory/core/annotations/model.py +334 -0
  35. data_factory/core/annotations/voc.py +231 -0
  36. data_factory/core/annotations/yolo.py +283 -0
  37. data_factory/core/bktree.py +89 -0
  38. data_factory/core/fs.py +245 -0
  39. data_factory/core/imaging.py +626 -0
  40. data_factory/core/ops/__init__.py +44 -0
  41. data_factory/core/ops/analyze.py +383 -0
  42. data_factory/core/ops/annotate_export.py +326 -0
  43. data_factory/core/ops/audit.py +274 -0
  44. data_factory/core/ops/augment.py +454 -0
  45. data_factory/core/ops/collect.py +241 -0
  46. data_factory/core/ops/combine.py +582 -0
  47. data_factory/core/ops/convert.py +212 -0
  48. data_factory/core/ops/dataset.py +246 -0
  49. data_factory/core/ops/dedup.py +235 -0
  50. data_factory/core/ops/export.py +168 -0
  51. data_factory/core/ops/image.py +407 -0
  52. data_factory/core/ops/labels.py +215 -0
  53. data_factory/core/ops/pairing.py +133 -0
  54. data_factory/core/ops/rename.py +154 -0
  55. data_factory/core/ops/report.py +323 -0
  56. data_factory/core/ops/visualize.py +164 -0
  57. data_factory/core/pairs.py +144 -0
  58. data_factory/core/parallel.py +191 -0
  59. data_factory/core/provenance.py +136 -0
  60. data_factory/core/registry.py +339 -0
  61. data_factory/core/stats.py +403 -0
  62. data_factory/core/types.py +148 -0
  63. data_factory/core/writer.py +194 -0
  64. data_factory/desktop.py +207 -0
  65. data_factory/hardware.py +243 -0
  66. data_factory/pipeline.py +192 -0
  67. data_factory/py.typed +0 -0
  68. data_factory/service/__init__.py +26 -0
  69. data_factory/service/app.py +483 -0
  70. data_factory/service/binding.py +174 -0
  71. data_factory/service/deps.py +133 -0
  72. data_factory/service/jobs.py +177 -0
  73. data_factory/service/limits.py +71 -0
  74. data_factory/service/routers/__init__.py +11 -0
  75. data_factory/service/routers/annotate.py +582 -0
  76. data_factory/service/routers/local.py +464 -0
  77. data_factory/service/routers/media.py +154 -0
  78. data_factory/service/routers/predictions.py +116 -0
  79. data_factory/service/routers/promote.py +182 -0
  80. data_factory/service/routers/tools.py +308 -0
  81. data_factory/service/static/annotate/api.js +246 -0
  82. data_factory/service/static/annotate/app.js +1696 -0
  83. data_factory/service/static/annotate/assist.js +106 -0
  84. data_factory/service/static/annotate/filmstrip.js +286 -0
  85. data_factory/service/static/annotate/hit.js +146 -0
  86. data_factory/service/static/annotate/labels.js +197 -0
  87. data_factory/service/static/annotate/render.js +286 -0
  88. data_factory/service/static/annotate/scene.js +277 -0
  89. data_factory/service/static/annotate/shapelist.js +72 -0
  90. data_factory/service/static/annotate/tools/box.js +96 -0
  91. data_factory/service/static/annotate/tools/line.js +96 -0
  92. data_factory/service/static/annotate/tools/obb.js +88 -0
  93. data_factory/service/static/annotate/tools/point.js +49 -0
  94. data_factory/service/static/annotate/tools/polygon.js +110 -0
  95. data_factory/service/static/annotate/tools/select.js +227 -0
  96. data_factory/service/static/annotate/undo.js +176 -0
  97. data_factory/service/static/annotate/viewport.js +156 -0
  98. data_factory/service/static/annotate.html +390 -0
  99. data_factory/service/static/index.html +632 -0
  100. data_factory/service/workspace.py +244 -0
  101. easyannotate/__init__.py +64 -0
  102. easyannotate/__main__.py +5 -0
  103. easyannotate/project.py +181 -0
  104. easyannotate/py.typed +0 -0
  105. easyannotate-0.4.0.dist-info/METADATA +767 -0
  106. easyannotate-0.4.0.dist-info/RECORD +109 -0
  107. easyannotate-0.4.0.dist-info/WHEEL +4 -0
  108. easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
  109. easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,323 @@
1
+ """A standalone HTML report for a dataset.
2
+
3
+ A JSON payload is what a program wants. A person deciding whether a dataset is
4
+ ready to train wants to *see* the class imbalance, and wants to be able to send
5
+ that to a colleague. This writes one self-contained file -- no CDN, no assets
6
+ folder, no server -- that opens in any browser and survives being emailed.
7
+
8
+ Charts are hand-written SVG for the same reason: a report that depends on a
9
+ charting library downloading at open time is a report that is blank in six
10
+ months, or on a machine behind a firewall.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import html
16
+ import json
17
+ from pathlib import Path
18
+ from typing import Any
19
+
20
+ from .. import stats
21
+ from ..annotations import formats
22
+ from ..annotations.model import AnnotationError
23
+ from ..registry import Param, operation
24
+ from ..types import Action, Context, OpResult
25
+
26
+ REPORT_PARAMS = (
27
+ Param("folder", "dir", "Dataset to report on.", required=True, must_exist=True),
28
+ Param("output", "file", "Where to write the .html file.", default="dataset-report.html"),
29
+ Param(
30
+ "source_format",
31
+ "choice",
32
+ "Input format; 'auto' sniffs the layout.",
33
+ default="auto",
34
+ choices=("auto",) + formats.FORMATS,
35
+ ),
36
+ Param("title", "str", "Heading shown at the top of the report.", default=""),
37
+ Param("recursive", "bool", "Include sub-folders.", default=True),
38
+ Param("json_too", "bool", "Also write the raw statistics as .json beside it.", default=False),
39
+ )
40
+
41
+
42
+ @operation(
43
+ "report.html",
44
+ title="Dataset report",
45
+ category="report",
46
+ summary="Write a self-contained HTML report: class balance, object sizes, findings and fixes.",
47
+ params=REPORT_PARAMS,
48
+ destructive=True,
49
+ notes=(
50
+ "One file, no dependencies, opens anywhere. Contains the same findings\n"
51
+ "inspect.health prints, plus the distributions that are hard to read as\n"
52
+ "numbers -- class balance and object size especially.\n"
53
+ "\n"
54
+ "Written for handing over: attach it to the ticket that says the dataset\n"
55
+ "is ready, and the reviewer can see for themselves whether it is."
56
+ ),
57
+ )
58
+ def html_report(params: dict[str, Any], ctx: Context) -> OpResult:
59
+ result = OpResult(op="report.html", dry_run=ctx.dry_run)
60
+ folder: Path = params["folder"]
61
+ output: Path = Path(params["output"])
62
+
63
+ try:
64
+ dataset = formats.read_dataset(
65
+ folder, params["source_format"], recursive=params["recursive"], ctx=ctx
66
+ )
67
+ except AnnotationError as exc:
68
+ result.fail(str(exc))
69
+ return result
70
+
71
+ ctx.emit(1, 2, "analysing")
72
+ payload = stats.analyse(dataset)
73
+ # The folder name, not the full path. A report is made to be shared, and on
74
+ # a hosted service the absolute path leaks the server's layout and the
75
+ # dataset's private id to anyone the file is forwarded to. The full path is
76
+ # still in the JSON sidecar for whoever actually needs it.
77
+ payload["source"] = folder.name or str(folder)
78
+ ctx.emit(2, 2, "rendering")
79
+
80
+ heading = params["title"] or folder.name
81
+ document = _render(heading, payload)
82
+
83
+ result.record(Action.WRITE, output, output, "html report")
84
+ if not ctx.dry_run:
85
+ output.parent.mkdir(parents=True, exist_ok=True)
86
+ output.write_text(document, encoding="utf-8")
87
+ if params["json_too"]:
88
+ json_path = output.with_suffix(".json")
89
+ json_path.write_text(
90
+ json.dumps({**payload, "source_path": str(folder)}, indent=2), encoding="utf-8"
91
+ )
92
+ result.record(Action.WRITE, json_path, json_path, "raw statistics")
93
+
94
+ totals = payload["totals"]
95
+ result.counts.update(
96
+ {
97
+ "images": totals["images"],
98
+ "objects": totals["objects"],
99
+ "classes": totals["classes"],
100
+ "findings": len(payload["findings"]),
101
+ }
102
+ )
103
+ result.warnings.extend(dataset.warnings[:10])
104
+ result.data.update(
105
+ {
106
+ "report": output,
107
+ "size_bytes": len(document.encode("utf-8")),
108
+ "findings": payload["findings"],
109
+ }
110
+ )
111
+ return result
112
+
113
+
114
+ # ------------------------------------------------------------------ render
115
+
116
+ _CSS = """
117
+ :root {
118
+ --bg: #f7f8fa; --panel: #ffffff; --ink: #1a1d23; --muted: #5c6470;
119
+ --line: #e3e6eb; --accent: #0b6ef4; --amber: #d98200; --red: #c8352c;
120
+ --green: #1c8a5a;
121
+ }
122
+ @media (prefers-color-scheme: dark) {
123
+ :root {
124
+ --bg: #14161a; --panel: #1c1f25; --ink: #e8eaee; --muted: #9aa3b0;
125
+ --line: #2b2f37; --accent: #5aa2ff; --amber: #f0a72a; --red: #ff6b60;
126
+ --green: #48c98a;
127
+ }
128
+ }
129
+ * { box-sizing: border-box; }
130
+ body {
131
+ margin: 0; padding: 2.5rem 1.5rem; background: var(--bg); color: var(--ink);
132
+ font: 15px/1.6 -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
133
+ }
134
+ .wrap { max-width: 62rem; margin: 0 auto; }
135
+ h1 { font-size: 1.7rem; margin: 0 0 .25rem; letter-spacing: -.02em; }
136
+ h2 { font-size: 1.05rem; margin: 2.5rem 0 .9rem; text-transform: uppercase;
137
+ letter-spacing: .08em; color: var(--muted); font-weight: 600; }
138
+ .sub { color: var(--muted); margin: 0 0 2rem; font-size: .9rem; }
139
+ .cards { display: grid; gap: .75rem; grid-template-columns: repeat(auto-fit, minmax(8.5rem, 1fr)); }
140
+ .card { background: var(--panel); border: 1px solid var(--line); border-radius: 10px; padding: 1rem; }
141
+ .card .n { font-size: 1.75rem; font-weight: 650; letter-spacing: -.02em; }
142
+ .card .k { color: var(--muted); font-size: .78rem; text-transform: uppercase;
143
+ letter-spacing: .06em; margin-top: .15rem; }
144
+ .panel { background: var(--panel); border: 1px solid var(--line); border-radius: 10px;
145
+ padding: 1.25rem; overflow-x: auto; }
146
+ table { border-collapse: collapse; width: 100%; font-size: .9rem; }
147
+ th, td { text-align: left; padding: .5rem .6rem; border-bottom: 1px solid var(--line); }
148
+ th { color: var(--muted); font-weight: 600; font-size: .78rem;
149
+ text-transform: uppercase; letter-spacing: .05em; }
150
+ td.num, th.num { text-align: right; font-variant-numeric: tabular-nums; }
151
+ tr:last-child td { border-bottom: 0; }
152
+ .finding { background: var(--panel); border: 1px solid var(--line); border-left: 3px solid var(--line);
153
+ border-radius: 8px; padding: .9rem 1.1rem; margin-bottom: .6rem; }
154
+ .finding.error { border-left-color: var(--red); }
155
+ .finding.warning { border-left-color: var(--amber); }
156
+ .finding.note { border-left-color: var(--accent); }
157
+ .finding .t { font-weight: 620; }
158
+ .finding .d { color: var(--muted); font-size: .9rem; margin-top: .2rem; }
159
+ .finding .f { font-size: .9rem; margin-top: .5rem; }
160
+ .finding .f b { font-weight: 600; }
161
+ .tag { display: inline-block; font-size: .68rem; text-transform: uppercase;
162
+ letter-spacing: .07em; padding: .1rem .45rem; border-radius: 4px;
163
+ vertical-align: .12em; margin-right: .5rem; font-weight: 700; color: #fff; }
164
+ .tag.error { background: var(--red); } .tag.warning { background: var(--amber); }
165
+ .tag.note { background: var(--accent); }
166
+ .bar { height: 1.4rem; border-radius: 3px; background: var(--accent); min-width: 2px; }
167
+ .ok { color: var(--green); font-weight: 600; }
168
+ footer { margin-top: 3rem; color: var(--muted); font-size: .8rem;
169
+ border-top: 1px solid var(--line); padding-top: 1rem; }
170
+ code { font-family: ui-monospace, "Cascadia Code", Consolas, monospace; font-size: .85em;
171
+ background: rgba(128,128,128,.14); padding: .1rem .35rem; border-radius: 4px; }
172
+ """
173
+
174
+
175
+ def _render(heading: str, payload: dict[str, Any]) -> str:
176
+ totals = payload["totals"]
177
+ parts: list[str] = []
178
+ add = parts.append
179
+
180
+ add("<!doctype html><html lang='en'><head><meta charset='utf-8'>")
181
+ add("<meta name='viewport' content='width=device-width,initial-scale=1'>")
182
+ add(f"<title>{_e(heading)} — dataset report</title>")
183
+ add(f"<style>{_CSS}</style></head><body><div class='wrap'>")
184
+
185
+ add(f"<h1>{_e(heading)}</h1>")
186
+ add(
187
+ f"<p class='sub'>{_e(str(payload.get('source', '')))} &middot; "
188
+ f"detected format <code>{_e(payload.get('source_format', '?'))}</code></p>"
189
+ )
190
+
191
+ add("<div class='cards'>")
192
+ for key, label in (
193
+ ("images", "images"),
194
+ ("objects", "objects"),
195
+ ("classes", "classes"),
196
+ ("backgrounds", "backgrounds"),
197
+ ("polygon_objects", "polygons"),
198
+ ):
199
+ add(f"<div class='card'><div class='n'>{totals.get(key, 0):,}</div>"
200
+ f"<div class='k'>{label}</div></div>")
201
+ add("</div>")
202
+
203
+ # -- findings
204
+ findings = payload["findings"]
205
+ add("<h2>Findings</h2>")
206
+ if not findings:
207
+ add("<div class='panel'><span class='ok'>Nothing to flag.</span> "
208
+ "Class balance, object sizes and geometry all look reasonable.</div>")
209
+ for finding in findings:
210
+ severity = _e(finding["severity"])
211
+ add(
212
+ f"<div class='finding {severity}'>"
213
+ f"<div class='t'><span class='tag {severity}'>{severity}</span>"
214
+ f"{_e(finding['issue'])}</div>"
215
+ f"<div class='d'>{_e(finding['detail'])}</div>"
216
+ f"<div class='f'><b>Fix:</b> {_e(finding['fix'])}</div></div>"
217
+ )
218
+
219
+ # -- class balance
220
+ classes = payload["per_class"]
221
+ add("<h2>Class balance</h2><div class='panel'>")
222
+ if classes:
223
+ balance = payload["balance"]
224
+ add(
225
+ f"<p class='sub' style='margin:0 0 1rem'>Commonest class outnumbers the "
226
+ f"rarest by <b>{balance['ratio']:g}&times;</b> "
227
+ f"(imbalance index {balance['gini']:g}, 0 = even).</p>"
228
+ )
229
+ peak = max(c["objects"] for c in classes)
230
+ add("<table><tr><th>Class</th><th class='num'>Objects</th>"
231
+ "<th class='num'>Images</th><th class='num'>Share</th>"
232
+ "<th style='width:38%'>Distribution</th></tr>")
233
+ for entry in classes:
234
+ width = max(1, round(100 * entry["objects"] / peak))
235
+ add(
236
+ f"<tr><td>{_e(entry['class'])}</td>"
237
+ f"<td class='num'>{entry['objects']:,}</td>"
238
+ f"<td class='num'>{entry['images']:,}</td>"
239
+ f"<td class='num'>{entry['share']:.1%}</td>"
240
+ f"<td><div class='bar' style='width:{width}%'></div></td></tr>"
241
+ )
242
+ add("</table>")
243
+ else:
244
+ add("<p>No classes found.</p>")
245
+ add("</div>")
246
+
247
+ # -- object sizes
248
+ sizes = payload["object_sizes"]
249
+ total_objects = max(1, sum(sizes.values()))
250
+ add("<h2>Object size</h2><div class='panel'>")
251
+ add("<table><tr><th>Bucket</th><th class='num'>Objects</th><th class='num'>Share</th>"
252
+ "<th style='width:38%'></th></tr>")
253
+ explanations = {
254
+ "tiny": f"under {stats.TINY_FRACTION:.2%} of the frame",
255
+ "small": f"under {stats.SMALL_AREA:,} px&sup2; (COCO 'small')",
256
+ "medium": f"under {stats.MEDIUM_AREA:,} px&sup2;",
257
+ "large": "everything bigger",
258
+ }
259
+ for bucket in ("tiny", "small", "medium", "large"):
260
+ count = sizes.get(bucket, 0)
261
+ share = count / total_objects
262
+ add(
263
+ f"<tr><td><b>{bucket}</b><br><span class='k' style='font-size:.78rem;color:var(--muted)'>"
264
+ f"{explanations[bucket]}</span></td>"
265
+ f"<td class='num'>{count:,}</td><td class='num'>{share:.1%}</td>"
266
+ f"<td><div class='bar' style='width:{max(1, round(100 * share))}%'></div></td></tr>"
267
+ )
268
+ add("</table></div>")
269
+
270
+ # -- shape of the data
271
+ area = payload["relative_area"]
272
+ ratio = payload["aspect_ratio"]
273
+ per_image = payload["objects_per_image"]
274
+ add("<h2>Distributions</h2><div class='panel'><table>")
275
+ add("<tr><th>Measure</th><th class='num'>Min</th><th class='num'>p25</th>"
276
+ "<th class='num'>Median</th><th class='num'>p75</th><th class='num'>Max</th></tr>")
277
+ add(
278
+ "<tr><td>Object area, as a fraction of its image</td>"
279
+ + "".join(
280
+ f"<td class='num'>{area[k]:.4f}</td>" for k in ("min", "p25", "median", "p75", "max")
281
+ )
282
+ + "</tr>"
283
+ )
284
+ add(
285
+ "<tr><td>Object aspect ratio (width / height)</td>"
286
+ + "".join(
287
+ f"<td class='num'>{ratio[k]:.2f}</td>" for k in ("min", "p25", "median", "p75", "max")
288
+ )
289
+ + "</tr>"
290
+ )
291
+ add("</table>")
292
+ add(
293
+ f"<p class='sub' style='margin:1rem 0 0'>Objects per image: mean "
294
+ f"<b>{per_image['mean']}</b>, max <b>{per_image['max']}</b>.</p>"
295
+ )
296
+ add("</div>")
297
+
298
+ # -- composition
299
+ add("<h2>Composition</h2><div class='panel'><table>")
300
+ add("<tr><th>Split</th><th class='num'>Images</th></tr>")
301
+ for name, count in payload["splits"].items():
302
+ add(f"<tr><td>{_e(name)}</td><td class='num'>{count:,}</td></tr>")
303
+ add("</table>")
304
+ image_sizes = payload["image_sizes"]
305
+ add(f"<p class='sub' style='margin:1rem 0 .4rem'>{image_sizes['distinct']} distinct "
306
+ f"image resolution(s). Most common:</p><table>")
307
+ add("<tr><th>Resolution</th><th class='num'>Images</th></tr>")
308
+ for entry in image_sizes["most_common"]:
309
+ add(f"<tr><td><code>{_e(entry['size'])}</code></td>"
310
+ f"<td class='num'>{entry['images']:,}</td></tr>")
311
+ add("</table></div>")
312
+
313
+ add(
314
+ "<footer>Generated by Data Factory <code>report.html</code>. "
315
+ "Thresholds for 'tiny', 'imbalanced' and 'too few examples' are documented "
316
+ "in <code>data_factory/core/stats.py</code>.</footer>"
317
+ )
318
+ add("</div></body></html>")
319
+ return "".join(parts)
320
+
321
+
322
+ def _e(value: Any) -> str:
323
+ return html.escape(str(value), quote=True)
@@ -0,0 +1,164 @@
1
+ """Draw annotations onto copies of the images, for eyeballing a dataset.
2
+
3
+ Merges three near-identical scripts. Colour is now derived from the class name
4
+ rather than fixed green, so a mislabelled object is visible at a glance instead
5
+ of requiring you to read every caption.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ from ..annotations import voc
14
+ from ..annotations.model import Annotation, AnnotationError
15
+ from ..pairs import PairIndex
16
+ from ..registry import Param, operation
17
+ from ..types import Action, Context, OpResult
18
+
19
+ #: Distinct, reasonably colour-blind-friendly BGR values (OpenCV order).
20
+ PALETTE = (
21
+ (255, 176, 0), (0, 200, 255), (80, 220, 80), (200, 80, 220),
22
+ (60, 60, 240), (240, 220, 0), (140, 140, 255), (0, 140, 255),
23
+ )
24
+
25
+ PARAMS = (
26
+ Param("folder", "dir", "Folder of image/label pairs.", required=True, must_exist=True),
27
+ Param("output", "dir", "Where annotated copies are written.", required=True),
28
+ Param("draw", "choice", "Which shapes to render.",
29
+ default="auto", choices=("auto", "bbox", "polygon", "both")),
30
+ Param("split_by_type", "bool",
31
+ "Route polygon-annotated and bbox-only images into separate sub-folders.",
32
+ default=False),
33
+ Param("thickness", "int", "Line thickness.", default=2, minimum=1, maximum=10),
34
+ Param("font_scale", "float", "Caption size.", default=0.55, minimum=0.1, maximum=3.0),
35
+ Param("show_labels", "bool", "Draw the class name next to each shape.", default=True),
36
+ Param("limit", "int", "Stop after this many images (0 = all). Handy for a quick look.",
37
+ default=0, minimum=0),
38
+ Param("recursive", "bool", "Include sub-folders.", default=False),
39
+ Param("label_ext", "str", "Sidecar label extension.", default=".xml"),
40
+ )
41
+
42
+
43
+ @operation(
44
+ "verify.visualize",
45
+ title="Render annotations onto images",
46
+ category="verify",
47
+ summary="Write a copy of each image with its boxes and polygons drawn on, colour-coded by class.",
48
+ params=PARAMS,
49
+ notes=(
50
+ "Non-destructive: the source folder is never touched.\n\n"
51
+ "draw=auto renders polygons where an object has one and falls back to its\n"
52
+ "box otherwise, which is the honest view of a mixed dataset. Use\n"
53
+ "split_by_type to see at a glance how much of the set has masks."
54
+ ),
55
+ )
56
+ def visualize(params: dict[str, Any], ctx: Context) -> OpResult:
57
+ result = OpResult(op="verify.visualize", dry_run=ctx.dry_run)
58
+ cv2, np = _load_cv2()
59
+
60
+ folder: Path = params["folder"]
61
+ output: Path = params["output"]
62
+ mode: str = params["draw"]
63
+ limit = int(params["limit"])
64
+
65
+ index = PairIndex.scan(folder, recursive=params["recursive"], label_ext=params["label_ext"])
66
+ pairs = index.complete()
67
+ if limit:
68
+ pairs = pairs[:limit]
69
+ result.counts["scanned"] = len(index)
70
+
71
+ for position, pair in enumerate(pairs, start=1):
72
+ ctx.check_cancelled()
73
+ ctx.emit(position, len(pairs), pair.stem)
74
+ assert pair.image is not None and pair.label is not None
75
+
76
+ try:
77
+ annotation = voc.read(pair.label)
78
+ except AnnotationError as exc:
79
+ result.warnings.append(f"{pair.label.name}: {exc}")
80
+ result.bump("unreadable_labels")
81
+ continue
82
+
83
+ image = cv2.imread(str(pair.image))
84
+ if image is None:
85
+ result.warnings.append(f"{pair.image.name}: OpenCV could not decode this image")
86
+ result.bump("unreadable_images")
87
+ continue
88
+
89
+ drawn = _render(cv2, np, image, annotation, params)
90
+ bucket = ""
91
+ if params["split_by_type"]:
92
+ bucket = "polygons" if annotation.has_polygons else "bbox_only"
93
+ result.bump(bucket)
94
+
95
+ destination = output / bucket / pair.image.name if bucket else output / pair.image.name
96
+ if not ctx.dry_run:
97
+ destination.parent.mkdir(parents=True, exist_ok=True)
98
+ cv2.imwrite(str(destination), drawn)
99
+ result.record(Action.WRITE, pair.image, destination, f"{len(annotation.shapes)} shape(s)")
100
+ result.bump("rendered")
101
+ if annotation.is_background:
102
+ result.bump("background_images")
103
+
104
+ result.data["output"] = output
105
+ result.data["mode"] = mode
106
+ return result
107
+
108
+
109
+ def _render(cv2: Any, np: Any, image: Any, annotation: Annotation, params: dict[str, Any]) -> Any:
110
+ canvas = image.copy()
111
+ thickness = int(params["thickness"])
112
+ scale = float(params["font_scale"])
113
+ mode = params["draw"]
114
+
115
+ for shape in annotation.shapes:
116
+ colour = colour_for(shape.label)
117
+ want_polygon = mode in ("polygon", "both") or (mode == "auto" and shape.has_polygon)
118
+ want_bbox = mode in ("bbox", "both") or (mode == "auto" and not shape.has_polygon)
119
+
120
+ if want_polygon and shape.has_polygon:
121
+ assert shape.polygon is not None
122
+ points = np.array(shape.polygon, np.int32).reshape((-1, 1, 2))
123
+ cv2.polylines(canvas, [points], isClosed=True, color=colour, thickness=thickness)
124
+ if want_bbox:
125
+ xmin, ymin, xmax, ymax = shape.bbox
126
+ cv2.rectangle(canvas, (xmin, ymin), (xmax, ymax), colour, thickness)
127
+
128
+ if params["show_labels"]:
129
+ _caption(cv2, canvas, shape.label, shape.bbox, colour, scale)
130
+
131
+ if annotation.is_background:
132
+ cv2.putText(
133
+ canvas, "BACKGROUND", (10, 26), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (60, 60, 240), 2, cv2.LINE_AA
134
+ )
135
+ return canvas
136
+
137
+
138
+ def _caption(
139
+ cv2: Any, canvas: Any, text: str, bbox: tuple[int, int, int, int], colour: Any, scale: float
140
+ ) -> None:
141
+ xmin, ymin = bbox[0], bbox[1]
142
+ (width, height), _ = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, scale, 1)
143
+ # Flip the caption inside the box when the object sits against the top edge,
144
+ # so it never gets clipped off the canvas.
145
+ top = ymin - height - 6 if ymin - height - 6 > 0 else ymin + 2
146
+ cv2.rectangle(canvas, (xmin, top), (xmin + width + 8, top + height + 6), colour, -1)
147
+ cv2.putText(
148
+ canvas, text, (xmin + 4, top + height + 1),
149
+ cv2.FONT_HERSHEY_SIMPLEX, scale, (20, 20, 20), 1, cv2.LINE_AA,
150
+ )
151
+
152
+
153
+ def colour_for(label: str) -> tuple[int, int, int]:
154
+ """Stable colour per class name, so a class looks the same in every image."""
155
+ return PALETTE[sum(ord(c) for c in label) % len(PALETTE)]
156
+
157
+
158
+ def _load_cv2() -> tuple[Any, Any]:
159
+ try:
160
+ import cv2
161
+ import numpy as np
162
+ except ImportError as exc: # pragma: no cover - depends on environment
163
+ raise RuntimeError(f"this operation needs opencv-python and numpy ({exc})") from exc
164
+ return cv2, np
@@ -0,0 +1,144 @@
1
+ """The image + sidecar-label pair.
2
+
3
+ Nearly every original script re-implemented this: loop the XMLs, try each image
4
+ extension in turn, remember to move both halves together. Getting it wrong is
5
+ how datasets end up with orphans. It lives here once, and every operation
6
+ consumes it, so "keep the pair together" stops being a thing anyone can forget.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from collections.abc import Iterable, Iterator
12
+ from dataclasses import dataclass
13
+ from pathlib import Path
14
+
15
+ from . import fs
16
+
17
+
18
+ @dataclass(slots=True)
19
+ class Pair:
20
+ """One dataset item: an image, its label file, or a half of either."""
21
+
22
+ key: str
23
+ """Unique identity: the stem, prefixed by sub-folder when scanning recursively."""
24
+
25
+ stem: str
26
+ image: Path | None = None
27
+ label: Path | None = None
28
+
29
+ @property
30
+ def complete(self) -> bool:
31
+ return self.image is not None and self.label is not None
32
+
33
+ @property
34
+ def image_only(self) -> bool:
35
+ """An image with no annotation -- unlabelled, not necessarily junk."""
36
+ return self.image is not None and self.label is None
37
+
38
+ @property
39
+ def label_only(self) -> bool:
40
+ """An annotation whose image is gone -- always junk."""
41
+ return self.image is None and self.label is not None
42
+
43
+ @property
44
+ def files(self) -> list[Path]:
45
+ return [p for p in (self.image, self.label) if p is not None]
46
+
47
+ @property
48
+ def anchor(self) -> Path:
49
+ """Whichever half exists, for reporting and sorting."""
50
+ anchor = self.image or self.label
51
+ assert anchor is not None, "a Pair must hold at least one file"
52
+ return anchor
53
+
54
+
55
+ class PairIndex:
56
+ """A scanned folder, resolved into pairs.
57
+
58
+ Scanning is done once and reused; operations that need both the image and
59
+ the label (rename, split, dedup, sort) then cannot desynchronise them.
60
+ """
61
+
62
+ def __init__(self, root: Path, pairs: list[Pair]) -> None:
63
+ self.root = root
64
+ self.pairs = pairs
65
+
66
+ # -- construction ------------------------------------------------------
67
+ @classmethod
68
+ def scan(
69
+ cls,
70
+ root: Path,
71
+ *,
72
+ recursive: bool = False,
73
+ image_exts: Iterable[str] = fs.IMAGE_EXTS,
74
+ label_ext: str = fs.DEFAULT_LABEL_EXT,
75
+ ) -> PairIndex:
76
+ extset = {e.lower() for e in image_exts}
77
+ label_ext = label_ext.lower()
78
+ found: dict[str, Pair] = {}
79
+
80
+ for path in fs.iter_files(root, recursive=recursive):
81
+ suffix = path.suffix.lower()
82
+ is_img = suffix in extset
83
+ is_label = suffix == label_ext
84
+ if not (is_img or is_label):
85
+ continue
86
+
87
+ # Key on sub-folder + stem so a recursive scan of train/ and val/
88
+ # does not fuse two distinct images that happen to share a name.
89
+ relative = fs.relative_to_safe(path, root)
90
+ key = str(relative.parent / relative.stem).replace("\\", "/")
91
+ pair = found.get(key)
92
+ if pair is None:
93
+ pair = Pair(key=key, stem=path.stem)
94
+ found[key] = pair
95
+
96
+ if is_img:
97
+ # Deterministic winner when both cat.jpg and cat.png exist, so
98
+ # the un-chosen twin surfaces as a duplicate rather than
99
+ # silently shadowing whichever the OS listed first.
100
+ if pair.image is None or _ext_rank(path, extset) < _ext_rank(pair.image, extset):
101
+ pair.image = path
102
+ else:
103
+ pair.label = path
104
+
105
+ return cls(root, [found[k] for k in sorted(found)])
106
+
107
+ # -- views -------------------------------------------------------------
108
+ def complete(self) -> list[Pair]:
109
+ return [p for p in self.pairs if p.complete]
110
+
111
+ def image_only(self) -> list[Pair]:
112
+ return [p for p in self.pairs if p.image_only]
113
+
114
+ def label_only(self) -> list[Pair]:
115
+ return [p for p in self.pairs if p.label_only]
116
+
117
+ def with_images(self) -> list[Pair]:
118
+ return [p for p in self.pairs if p.image is not None]
119
+
120
+ def summary(self) -> dict[str, int]:
121
+ return {
122
+ "pairs_total": len(self.pairs),
123
+ "complete": len(self.complete()),
124
+ "images_without_labels": len(self.image_only()),
125
+ "labels_without_images": len(self.label_only()),
126
+ }
127
+
128
+ def __iter__(self) -> Iterator[Pair]:
129
+ return iter(self.pairs)
130
+
131
+ def __len__(self) -> int:
132
+ return len(self.pairs)
133
+
134
+
135
+ def _ext_rank(path: Path, extset: set[str]) -> int:
136
+ """Rank extensions by the canonical order so the choice is reproducible."""
137
+ order = [e for e in (".jpg", ".jpeg", ".png", ".bmp", ".tif", ".tiff", ".webp") if e in extset]
138
+ suffix = path.suffix.lower()
139
+ return order.index(suffix) if suffix in order else len(order)
140
+
141
+
142
+ def sidecar_for(image: Path, label_ext: str = fs.DEFAULT_LABEL_EXT) -> Path:
143
+ """The label path an image would have, whether or not it exists."""
144
+ return image.with_suffix(label_ext)