easyannotate 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. data_factory/__init__.py +82 -0
  2. data_factory/__main__.py +6 -0
  3. data_factory/annotate/__init__.py +15 -0
  4. data_factory/annotate/attach.py +398 -0
  5. data_factory/annotate/demo.py +161 -0
  6. data_factory/annotate/images.py +443 -0
  7. data_factory/annotate/labels.py +399 -0
  8. data_factory/annotate/loop.py +359 -0
  9. data_factory/annotate/projects.py +207 -0
  10. data_factory/annotate/review.py +471 -0
  11. data_factory/annotate/schema.sql +171 -0
  12. data_factory/annotate/shapes.py +488 -0
  13. data_factory/annotate/store.py +292 -0
  14. data_factory/annotate/tools/__init__.py +39 -0
  15. data_factory/annotate/tools/fake.py +170 -0
  16. data_factory/annotate/tools/install.py +254 -0
  17. data_factory/annotate/tools/local.py +389 -0
  18. data_factory/annotate/tools/postprocess.py +175 -0
  19. data_factory/annotate/tools/protocol.py +238 -0
  20. data_factory/annotate/tools/registry.py +196 -0
  21. data_factory/annotate/tools/results.py +83 -0
  22. data_factory/annotate/tools/serve.py +106 -0
  23. data_factory/cli/__init__.py +21 -0
  24. data_factory/cli/main.py +837 -0
  25. data_factory/cli/menu.py +173 -0
  26. data_factory/cli/render.py +158 -0
  27. data_factory/core/__init__.py +11 -0
  28. data_factory/core/annotations/__init__.py +5 -0
  29. data_factory/core/annotations/coco.py +394 -0
  30. data_factory/core/annotations/csvfmt.py +174 -0
  31. data_factory/core/annotations/dota.py +98 -0
  32. data_factory/core/annotations/formats.py +626 -0
  33. data_factory/core/annotations/labelme.py +153 -0
  34. data_factory/core/annotations/model.py +334 -0
  35. data_factory/core/annotations/voc.py +231 -0
  36. data_factory/core/annotations/yolo.py +283 -0
  37. data_factory/core/bktree.py +89 -0
  38. data_factory/core/fs.py +245 -0
  39. data_factory/core/imaging.py +626 -0
  40. data_factory/core/ops/__init__.py +44 -0
  41. data_factory/core/ops/analyze.py +383 -0
  42. data_factory/core/ops/annotate_export.py +326 -0
  43. data_factory/core/ops/audit.py +274 -0
  44. data_factory/core/ops/augment.py +454 -0
  45. data_factory/core/ops/collect.py +241 -0
  46. data_factory/core/ops/combine.py +582 -0
  47. data_factory/core/ops/convert.py +212 -0
  48. data_factory/core/ops/dataset.py +246 -0
  49. data_factory/core/ops/dedup.py +235 -0
  50. data_factory/core/ops/export.py +168 -0
  51. data_factory/core/ops/image.py +407 -0
  52. data_factory/core/ops/labels.py +215 -0
  53. data_factory/core/ops/pairing.py +133 -0
  54. data_factory/core/ops/rename.py +154 -0
  55. data_factory/core/ops/report.py +323 -0
  56. data_factory/core/ops/visualize.py +164 -0
  57. data_factory/core/pairs.py +144 -0
  58. data_factory/core/parallel.py +191 -0
  59. data_factory/core/provenance.py +136 -0
  60. data_factory/core/registry.py +339 -0
  61. data_factory/core/stats.py +403 -0
  62. data_factory/core/types.py +148 -0
  63. data_factory/core/writer.py +194 -0
  64. data_factory/desktop.py +207 -0
  65. data_factory/hardware.py +243 -0
  66. data_factory/pipeline.py +192 -0
  67. data_factory/py.typed +0 -0
  68. data_factory/service/__init__.py +26 -0
  69. data_factory/service/app.py +483 -0
  70. data_factory/service/binding.py +174 -0
  71. data_factory/service/deps.py +133 -0
  72. data_factory/service/jobs.py +177 -0
  73. data_factory/service/limits.py +71 -0
  74. data_factory/service/routers/__init__.py +11 -0
  75. data_factory/service/routers/annotate.py +582 -0
  76. data_factory/service/routers/local.py +464 -0
  77. data_factory/service/routers/media.py +154 -0
  78. data_factory/service/routers/predictions.py +116 -0
  79. data_factory/service/routers/promote.py +182 -0
  80. data_factory/service/routers/tools.py +308 -0
  81. data_factory/service/static/annotate/api.js +246 -0
  82. data_factory/service/static/annotate/app.js +1696 -0
  83. data_factory/service/static/annotate/assist.js +106 -0
  84. data_factory/service/static/annotate/filmstrip.js +286 -0
  85. data_factory/service/static/annotate/hit.js +146 -0
  86. data_factory/service/static/annotate/labels.js +197 -0
  87. data_factory/service/static/annotate/render.js +286 -0
  88. data_factory/service/static/annotate/scene.js +277 -0
  89. data_factory/service/static/annotate/shapelist.js +72 -0
  90. data_factory/service/static/annotate/tools/box.js +96 -0
  91. data_factory/service/static/annotate/tools/line.js +96 -0
  92. data_factory/service/static/annotate/tools/obb.js +88 -0
  93. data_factory/service/static/annotate/tools/point.js +49 -0
  94. data_factory/service/static/annotate/tools/polygon.js +110 -0
  95. data_factory/service/static/annotate/tools/select.js +227 -0
  96. data_factory/service/static/annotate/undo.js +176 -0
  97. data_factory/service/static/annotate/viewport.js +156 -0
  98. data_factory/service/static/annotate.html +390 -0
  99. data_factory/service/static/index.html +632 -0
  100. data_factory/service/workspace.py +244 -0
  101. easyannotate/__init__.py +64 -0
  102. easyannotate/__main__.py +5 -0
  103. easyannotate/project.py +181 -0
  104. easyannotate/py.typed +0 -0
  105. easyannotate-0.4.0.dist-info/METADATA +767 -0
  106. easyannotate-0.4.0.dist-info/RECORD +109 -0
  107. easyannotate-0.4.0.dist-info/WHEEL +4 -0
  108. easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
  109. easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,231 @@
1
+ """Pascal VOC reading and writing.
2
+
3
+ The repository grew two incompatible polygon dialects: the polygon annotator
4
+ writes ``<polygon><x1>..<y1>..</polygon>`` while the SAM tool writes
5
+ ``<polygon><pt><x>..<y>..</pt></polygon>``. Only the first was ever readable by
6
+ the COCO converter, so SAM-produced masks silently converted to empty
7
+ segmentations. This reader accepts both -- plus plain ``<bndbox>`` -- and the
8
+ writer emits the indexed dialect as the canonical form.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import xml.etree.ElementTree as ET
14
+ from pathlib import Path
15
+ from xml.dom import minidom
16
+
17
+ from ..fs import write_text_atomic
18
+ from .model import Annotation, AnnotationError, Point, Shape
19
+
20
+
21
+ def read(path: Path) -> Annotation:
22
+ """Parse a Pascal VOC XML file into an :class:`Annotation`."""
23
+ try:
24
+ tree = ET.parse(path)
25
+ except ET.ParseError as exc:
26
+ raise AnnotationError(f"{path.name}: malformed XML ({exc})") from exc
27
+ except OSError as exc:
28
+ raise AnnotationError(f"{path.name}: cannot read ({exc})") from exc
29
+
30
+ root = tree.getroot()
31
+ width, height, depth = _read_size(root)
32
+ filename = _text(root, "filename") or path.with_suffix(".jpg").name
33
+
34
+ shapes: list[Shape] = []
35
+ for obj in root.findall("object"):
36
+ shape = _read_object(obj, path)
37
+ if shape is not None:
38
+ shapes.append(shape)
39
+
40
+ # Some tools omit <size>; recover it from the shapes so downstream COCO
41
+ # export is not left with a zero-dimension image.
42
+ if (width <= 0 or height <= 0) and shapes:
43
+ width = max(s.bbox[2] for s in shapes) + 1
44
+ height = max(s.bbox[3] for s in shapes) + 1
45
+
46
+ return Annotation(
47
+ filename=filename,
48
+ width=width,
49
+ height=height,
50
+ depth=depth,
51
+ shapes=shapes,
52
+ folder=_text(root, "folder") or "images",
53
+ source_path=path,
54
+ )
55
+
56
+
57
+ def _read_size(root: ET.Element) -> tuple[int, int, int]:
58
+ size = root.find("size")
59
+ if size is None:
60
+ return 0, 0, 3
61
+ return (
62
+ _int(_text(size, "width")),
63
+ _int(_text(size, "height")),
64
+ _int(_text(size, "depth")) or 3,
65
+ )
66
+
67
+
68
+ def _read_object(obj: ET.Element, path: Path) -> Shape | None:
69
+ label = _text(obj, "name")
70
+ if not label:
71
+ return None
72
+
73
+ polygon = _read_polygon(obj)
74
+ bbox = _read_bbox(obj)
75
+
76
+ if bbox is None and polygon:
77
+ # Polygon-only object (the SAM tool's output shape): derive the box so
78
+ # every consumer gets a bbox regardless of how it was annotated.
79
+ xs = [p[0] for p in polygon]
80
+ ys = [p[1] for p in polygon]
81
+ bbox = (min(xs), min(ys), max(xs), max(ys))
82
+ if bbox is None:
83
+ raise AnnotationError(f"{path.name}: object {label!r} has neither bndbox nor polygon")
84
+
85
+ xmin, ymin, xmax, ymax = bbox
86
+ if xmax < xmin:
87
+ xmin, xmax = xmax, xmin
88
+ if ymax < ymin:
89
+ ymin, ymax = ymax, ymin
90
+
91
+ return Shape(
92
+ label=label.strip(),
93
+ bbox=(xmin, ymin, xmax, ymax),
94
+ polygon=polygon,
95
+ pose=_text(obj, "pose") or "Unspecified",
96
+ truncated=_int(_text(obj, "truncated")),
97
+ difficult=_int(_text(obj, "difficult")),
98
+ occluded=_int(_text(obj, "occluded")),
99
+ confidence=_score(obj),
100
+ )
101
+
102
+
103
+ def _read_bbox(obj: ET.Element) -> tuple[int, int, int, int] | None:
104
+ node = obj.find("bndbox")
105
+ if node is None:
106
+ return None
107
+ try:
108
+ return (
109
+ _int(_text(node, "xmin")),
110
+ _int(_text(node, "ymin")),
111
+ _int(_text(node, "xmax")),
112
+ _int(_text(node, "ymax")),
113
+ )
114
+ except (TypeError, ValueError):
115
+ return None
116
+
117
+
118
+ def _read_polygon(obj: ET.Element) -> list[Point] | None:
119
+ """Read either polygon dialect, whichever this file happens to use."""
120
+ node = obj.find("polygon")
121
+ if node is None:
122
+ return None
123
+
124
+ # Dialect A -- indexed children: <x1>,<y1>,<x2>,<y2>,...
125
+ points: list[Point] = []
126
+ index = 1
127
+ while True:
128
+ x_node = node.find(f"x{index}")
129
+ y_node = node.find(f"y{index}")
130
+ if x_node is None or y_node is None:
131
+ break
132
+ points.append((_int(x_node.text), _int(y_node.text)))
133
+ index += 1
134
+ if points:
135
+ return points
136
+
137
+ # Dialect B -- repeated <pt><x/><y/></pt> elements (the SAM tool).
138
+ for pt in node.findall("pt"):
139
+ x_node, y_node = pt.find("x"), pt.find("y")
140
+ if x_node is None or y_node is None:
141
+ continue
142
+ points.append((_int(x_node.text), _int(y_node.text)))
143
+ return points or None
144
+
145
+
146
+ def write(annotation: Annotation, path: Path) -> Path:
147
+ """Serialise to Pascal VOC XML, emitting the indexed polygon dialect."""
148
+ root = ET.Element("annotation")
149
+ ET.SubElement(root, "folder").text = annotation.folder
150
+ ET.SubElement(root, "filename").text = annotation.filename
151
+ ET.SubElement(root, "path").text = annotation.filename
152
+
153
+ source = ET.SubElement(root, "source")
154
+ ET.SubElement(source, "database").text = "Unknown"
155
+
156
+ size = ET.SubElement(root, "size")
157
+ ET.SubElement(size, "width").text = str(annotation.width)
158
+ ET.SubElement(size, "height").text = str(annotation.height)
159
+ ET.SubElement(size, "depth").text = str(annotation.depth)
160
+
161
+ ET.SubElement(root, "segmented").text = "1" if annotation.has_polygons else "0"
162
+
163
+ for shape in annotation.shapes:
164
+ obj = ET.SubElement(root, "object")
165
+ ET.SubElement(obj, "name").text = shape.label
166
+ ET.SubElement(obj, "pose").text = shape.pose
167
+ ET.SubElement(obj, "truncated").text = str(shape.truncated)
168
+ ET.SubElement(obj, "difficult").text = str(shape.difficult)
169
+ ET.SubElement(obj, "occluded").text = str(shape.occluded)
170
+ # Written only when there is one, so a hand-drawn label produces
171
+ # byte-identical XML to before -- a <score>1.0</score> appearing on
172
+ # every ground-truth object would be a lie about where it came from.
173
+ if shape.confidence is not None:
174
+ ET.SubElement(obj, "score").text = f"{float(shape.confidence):.6f}"
175
+
176
+ bndbox = ET.SubElement(obj, "bndbox")
177
+ ET.SubElement(bndbox, "xmin").text = str(shape.bbox[0])
178
+ ET.SubElement(bndbox, "ymin").text = str(shape.bbox[1])
179
+ ET.SubElement(bndbox, "xmax").text = str(shape.bbox[2])
180
+ ET.SubElement(bndbox, "ymax").text = str(shape.bbox[3])
181
+
182
+ if shape.has_polygon:
183
+ assert shape.polygon is not None
184
+ polygon = ET.SubElement(obj, "polygon")
185
+ for index, (x, y) in enumerate(shape.polygon, start=1):
186
+ ET.SubElement(polygon, f"x{index}").text = str(int(x))
187
+ ET.SubElement(polygon, f"y{index}").text = str(int(y))
188
+
189
+ pretty = minidom.parseString(ET.tostring(root, encoding="unicode")).toprettyxml(indent=" ")
190
+ # Atomic: a half-written XML is an unparseable label, and the original
191
+ # is already gone by the time the write fails.
192
+ return write_text_atomic(path, pretty)
193
+
194
+
195
+ def write_background(filename: str, width: int, height: int, path: Path) -> Path:
196
+ """Write a valid, object-free annotation -- an explicit hard negative."""
197
+ return write(Annotation(filename=filename, width=width, height=height), path)
198
+
199
+
200
+ def _text(parent: ET.Element, tag: str) -> str | None:
201
+ node = parent.find(tag)
202
+ if node is None or node.text is None:
203
+ return None
204
+ return node.text.strip()
205
+
206
+
207
+ def _int(value: str | None) -> int:
208
+ """Parse a coordinate, tolerating the decimals some tools emit."""
209
+ if value is None:
210
+ return 0
211
+ try:
212
+ return int(float(value.strip()))
213
+ except (ValueError, AttributeError):
214
+ return 0
215
+
216
+
217
+ def _score(obj: ET.Element) -> float | None:
218
+ """A detector's confidence, if this object carries one.
219
+
220
+ Not part of the Pascal VOC spec, which predates anyone wanting to round
221
+ trip model output. Reading it is harmless -- files without it are
222
+ unaffected -- and it is what lets predictions survive a trip through the
223
+ format the rest of this tool already speaks.
224
+ """
225
+ raw = _text(obj, "score")
226
+ if raw is None:
227
+ return None
228
+ try:
229
+ return float(raw)
230
+ except (TypeError, ValueError):
231
+ return None
@@ -0,0 +1,283 @@
1
+ """YOLO (Darknet / Ultralytics) reading and writing.
2
+
3
+ A YOLO label is one ``.txt`` per image, coordinates normalised to 0-1, and a
4
+ class *index* rather than a name -- so the class list is external state. That is
5
+ the whole difficulty of this format: read a label file without the right
6
+ ``data.yaml`` and every class comes out wrong but nothing errors. We therefore
7
+ always carry the class list explicitly, and refuse to guess it.
8
+
9
+ Two line shapes are supported, which is how Ultralytics distinguishes tasks:
10
+
11
+ detection ``cls cx cy w h``
12
+ segmentation ``cls x1 y1 x2 y2 x3 y3 ...`` (>= 3 points)
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ from collections.abc import Iterable, Sequence
18
+ from dataclasses import dataclass, field
19
+ from pathlib import Path
20
+ from typing import Any
21
+
22
+ from ..fs import write_text_atomic
23
+ from .model import Annotation, AnnotationError, Point, Shape
24
+
25
+ #: Where Ultralytics expects things. ``images/x.jpg`` <-> ``labels/x.txt``.
26
+ IMAGE_DIR = "images"
27
+ LABEL_DIR = "labels"
28
+ DATA_YAML = "data.yaml"
29
+
30
+
31
+ @dataclass
32
+ class ClassList:
33
+ """The index-to-name table a YOLO dataset cannot be read without."""
34
+
35
+ names: list[str] = field(default_factory=list)
36
+ _index: dict[str, int] = field(default_factory=dict, init=False, repr=False)
37
+
38
+ def __post_init__(self) -> None:
39
+ self._index = {name: i for i, name in enumerate(self.names)}
40
+
41
+ @classmethod
42
+ def from_labels(cls, labels: Iterable[str]) -> ClassList:
43
+ """Build a stable list by sorting -- reproducible across runs."""
44
+ return cls(sorted({label.strip() for label in labels if label.strip()}))
45
+
46
+ def id_for(self, label: str) -> int:
47
+ if label not in self._index:
48
+ self._index[label] = len(self.names)
49
+ self.names.append(label)
50
+ return self._index[label]
51
+
52
+ def name_for(self, index: int) -> str:
53
+ if 0 <= index < len(self.names):
54
+ return self.names[index]
55
+ # Better a visibly wrong name than a silent misattribution to class 0.
56
+ return f"class_{index}"
57
+
58
+ def __len__(self) -> int:
59
+ return len(self.names)
60
+
61
+
62
+ def read(
63
+ path: Path,
64
+ classes: ClassList,
65
+ width: int,
66
+ height: int,
67
+ *,
68
+ filename: str | None = None,
69
+ ) -> Annotation:
70
+ """Parse one ``.txt`` into an :class:`Annotation` in absolute pixels."""
71
+ shapes: list[Shape] = []
72
+ try:
73
+ text = path.read_text(encoding="utf-8")
74
+ except OSError as exc:
75
+ raise AnnotationError(f"{path.name}: cannot read ({exc})") from exc
76
+
77
+ for number, line in enumerate(text.splitlines(), start=1):
78
+ line = line.strip()
79
+ if not line or line.startswith("#"):
80
+ continue
81
+ parts = line.split()
82
+ try:
83
+ class_id = int(float(parts[0]))
84
+ values = [float(v) for v in parts[1:]]
85
+ except (ValueError, IndexError) as exc:
86
+ raise AnnotationError(f"{path.name}:{number}: malformed line {line!r}") from exc
87
+
88
+ label = classes.name_for(class_id)
89
+ if len(values) == 4:
90
+ shapes.append(_box_shape(label, values, width, height))
91
+ elif len(values) == 5:
92
+ # `cls cx cy w h conf` -- what `--save-conf` writes, and therefore
93
+ # what comes back from a real inference run. Rejecting it meant the
94
+ # one file people most want to load was the one that would not
95
+ # parse. Five values are unambiguous: a polygon needs at least six
96
+ # and an even number of them.
97
+ shapes.append(_box_shape(label, values[:4], width, height, confidence=values[4]))
98
+ elif len(values) >= 6 and len(values) % 2 == 0:
99
+ shapes.append(_polygon_shape(label, values, width, height))
100
+ elif len(values) >= 7 and len(values) % 2 == 1:
101
+ # A segmentation line with a trailing confidence.
102
+ shapes.append(
103
+ _polygon_shape(label, values[:-1], width, height, confidence=values[-1])
104
+ )
105
+ else:
106
+ raise AnnotationError(
107
+ f"{path.name}:{number}: expected 4 box values (optionally plus a "
108
+ f"confidence) or an even number of >= 6 polygon values, got {len(values)}"
109
+ )
110
+
111
+ return Annotation(
112
+ filename=filename or path.with_suffix(".jpg").name,
113
+ width=width,
114
+ height=height,
115
+ shapes=shapes,
116
+ source_path=path,
117
+ )
118
+
119
+
120
+ def _box_shape(
121
+ label: str,
122
+ values: Sequence[float],
123
+ width: int,
124
+ height: int,
125
+ *,
126
+ confidence: float | None = None,
127
+ ) -> Shape:
128
+ cx, cy, bw, bh = values
129
+ half_w, half_h = bw * width / 2.0, bh * height / 2.0
130
+ centre_x, centre_y = cx * width, cy * height
131
+ return Shape(
132
+ label=label,
133
+ bbox=(
134
+ int(round(centre_x - half_w)),
135
+ int(round(centre_y - half_h)),
136
+ int(round(centre_x + half_w)),
137
+ int(round(centre_y + half_h)),
138
+ ),
139
+ confidence=confidence,
140
+ )
141
+
142
+
143
+ def _polygon_shape(
144
+ label: str,
145
+ values: Sequence[float],
146
+ width: int,
147
+ height: int,
148
+ *,
149
+ confidence: float | None = None,
150
+ ) -> Shape:
151
+ polygon: list[Point] = [
152
+ (int(round(values[i] * width)), int(round(values[i + 1] * height)))
153
+ for i in range(0, len(values), 2)
154
+ ]
155
+ xs = [p[0] for p in polygon]
156
+ ys = [p[1] for p in polygon]
157
+ return Shape(
158
+ label=label,
159
+ bbox=(min(xs), min(ys), max(xs), max(ys)),
160
+ polygon=polygon,
161
+ confidence=confidence,
162
+ # Eight values is YOLO-OBB's line, a rotated box by its corners. A
163
+ # genuine four-vertex segmentation reads the same way, and treating
164
+ # it as a rotated box costs nothing: it is one.
165
+ kind="obb" if len(polygon) == 4 else "",
166
+ )
167
+
168
+
169
+ def write(
170
+ annotation: Annotation,
171
+ path: Path,
172
+ classes: ClassList,
173
+ *,
174
+ segmentation: bool = True,
175
+ ) -> Path:
176
+ """Write one ``.txt``. Emits polygon lines when the shape has an outline.
177
+
178
+ A background image legitimately produces an empty file -- that is how YOLO
179
+ expresses a hard negative, and deleting it would silently drop the sample.
180
+ """
181
+ width = max(1, annotation.width)
182
+ height = max(1, annotation.height)
183
+ lines: list[str] = []
184
+
185
+ for shape in annotation.shapes:
186
+ class_id = classes.id_for(shape.label)
187
+ if segmentation and shape.has_polygon:
188
+ assert shape.polygon is not None
189
+ coords = [
190
+ f"{_unit(x / width):.6f} {_unit(y / height):.6f}" for x, y in shape.polygon
191
+ ]
192
+ lines.append(f"{class_id} " + " ".join(coords))
193
+ else:
194
+ xmin, ymin, xmax, ymax = shape.bbox
195
+ cx = _unit(((xmin + xmax) / 2.0) / width)
196
+ cy = _unit(((ymin + ymax) / 2.0) / height)
197
+ bw = _unit((xmax - xmin) / width)
198
+ bh = _unit((ymax - ymin) / height)
199
+ if bw <= 0 or bh <= 0:
200
+ continue
201
+ lines.append(f"{class_id} {cx:.6f} {cy:.6f} {bw:.6f} {bh:.6f}")
202
+
203
+ return write_text_atomic(path, "\n".join(lines) + ("\n" if lines else ""))
204
+
205
+
206
+ def _unit(value: float) -> float:
207
+ """YOLO coordinates outside 0-1 are rejected by most trainers."""
208
+ return max(0.0, min(1.0, value))
209
+
210
+
211
+ def write_data_yaml(
212
+ path: Path,
213
+ classes: ClassList,
214
+ *,
215
+ splits: dict[str, str] | None = None,
216
+ root: str = ".",
217
+ ) -> Path:
218
+ """Write the ``data.yaml`` that makes a folder a *dataset* to Ultralytics.
219
+
220
+ Hand-rolled rather than via PyYAML so the file keeps a readable, diffable
221
+ layout -- these get committed to repositories and reviewed.
222
+ """
223
+ splits = splits or {"train": "images"}
224
+ lines = [f"path: {root}"]
225
+ for key in ("train", "val", "test"):
226
+ if key in splits:
227
+ lines.append(f"{key}: {splits[key]}")
228
+ lines.append("")
229
+ lines.append(f"nc: {len(classes)}")
230
+ lines.append("names:")
231
+ for index, name in enumerate(classes.names):
232
+ lines.append(f" {index}: {name}")
233
+ return write_text_atomic(path, "\n".join(lines) + "\n")
234
+
235
+
236
+ def read_data_yaml(path: Path) -> tuple[ClassList, dict[str, str]]:
237
+ """Read a ``data.yaml``, accepting both the list and dict ``names`` styles."""
238
+ import yaml
239
+
240
+ try:
241
+ document: dict[str, Any] = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
242
+ except Exception as exc:
243
+ raise AnnotationError(f"{path.name}: cannot parse ({exc})") from exc
244
+
245
+ raw = document.get("names")
246
+ if isinstance(raw, dict):
247
+ # ``{0: cat, 1: dog}`` -- order by index, not by insertion.
248
+ names = [str(raw[k]) for k in sorted(raw, key=lambda k: int(k))]
249
+ elif isinstance(raw, list):
250
+ names = [str(n) for n in raw]
251
+ else:
252
+ raise AnnotationError(f"{path.name}: no usable 'names' entry")
253
+
254
+ splits = {k: str(document[k]) for k in ("train", "val", "test") if document.get(k)}
255
+ return ClassList(names), splits
256
+
257
+
258
+ def find_data_yaml(folder: Path) -> Path | None:
259
+ """Locate the class list for a folder, checking the usual parent layouts."""
260
+ for candidate in (folder, folder.parent, folder.parent.parent):
261
+ for name in (DATA_YAML, "dataset.yaml", "data.yml"):
262
+ path = candidate / name
263
+ if path.is_file():
264
+ return path
265
+ return None
266
+
267
+
268
+ def label_path_for(image: Path, root: Path) -> Path:
269
+ """The ``.txt`` matching an image, honouring the images/labels convention.
270
+
271
+ Falls back to a sidecar beside the image when the split layout is absent,
272
+ because plenty of real YOLO exports in the wild are simply flat folders.
273
+ """
274
+ try:
275
+ relative = image.relative_to(root)
276
+ except ValueError:
277
+ return image.with_suffix(".txt")
278
+ parts = list(relative.parts)
279
+ for index, part in enumerate(parts):
280
+ if part == IMAGE_DIR:
281
+ parts[index] = LABEL_DIR
282
+ return root.joinpath(*parts).with_suffix(".txt")
283
+ return image.with_suffix(".txt")
@@ -0,0 +1,89 @@
1
+ """A BK-tree for near-neighbour search in Hamming space.
2
+
3
+ Perceptual dedup needs "find everything within N bits of this hash". Doing that
4
+ by comparing every image to every kept image is O(n^2): on the 15k-image
5
+ Project_Door set that is ~112 million comparisons, which is why the original
6
+ script was only ever run on small folders.
7
+
8
+ A BK-tree exploits the triangle inequality -- Hamming distance is a metric, so
9
+ a child at distance k from a node can only contain matches within [d-r, d+r] of
10
+ the query. Whole subtrees get skipped, and lookup drops to roughly logarithmic
11
+ for the small radii (0-10 bits) that dedup actually uses.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from collections.abc import Callable, Iterator
17
+ from typing import Any
18
+
19
+
20
+ class _Node:
21
+ __slots__ = ("children", "item")
22
+
23
+ def __init__(self, item: Any) -> None:
24
+ self.item = item
25
+ self.children: dict[int, _Node] = {}
26
+
27
+
28
+ class BKTree:
29
+ """Metric tree supporting ``add`` and radius search.
30
+
31
+ ``distance`` must be a true metric (symmetric, triangle inequality) --
32
+ Hamming distance between equal-length hashes qualifies.
33
+ """
34
+
35
+ def __init__(self, distance: Callable[[Any, Any], int]) -> None:
36
+ self._distance = distance
37
+ self._root: _Node | None = None
38
+ self._size = 0
39
+
40
+ def add(self, item: Any) -> None:
41
+ self._size += 1
42
+ if self._root is None:
43
+ self._root = _Node(item)
44
+ return
45
+ node = self._root
46
+ while True:
47
+ gap = self._distance(item, node.item)
48
+ child = node.children.get(gap)
49
+ if child is None:
50
+ node.children[gap] = _Node(item)
51
+ return
52
+ node = child
53
+
54
+ def find(self, query: Any, radius: int) -> list[tuple[int, Any]]:
55
+ """Every stored item within ``radius``, as ``(distance, item)`` pairs."""
56
+ if self._root is None:
57
+ return []
58
+ matches: list[tuple[int, Any]] = []
59
+ stack = [self._root]
60
+ while stack:
61
+ node = stack.pop()
62
+ gap = self._distance(query, node.item)
63
+ if gap <= radius:
64
+ matches.append((gap, node.item))
65
+ # Triangle inequality: only these children can hold a match.
66
+ low, high = gap - radius, gap + radius
67
+ for edge, child in node.children.items():
68
+ if low <= edge <= high:
69
+ stack.append(child)
70
+ matches.sort(key=lambda pair: pair[0])
71
+ return matches
72
+
73
+ def nearest(self, query: Any, radius: int) -> tuple[int, Any] | None:
74
+ found = self.find(query, radius)
75
+ return found[0] if found else None
76
+
77
+ def __len__(self) -> int:
78
+ return self._size
79
+
80
+ def __iter__(self) -> Iterator[Any]:
81
+ if self._root is None:
82
+ return iter(())
83
+ items: list[Any] = []
84
+ stack = [self._root]
85
+ while stack:
86
+ node = stack.pop()
87
+ items.append(node.item)
88
+ stack.extend(node.children.values())
89
+ return iter(items)