easyannotate 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. data_factory/__init__.py +82 -0
  2. data_factory/__main__.py +6 -0
  3. data_factory/annotate/__init__.py +15 -0
  4. data_factory/annotate/attach.py +398 -0
  5. data_factory/annotate/demo.py +161 -0
  6. data_factory/annotate/images.py +443 -0
  7. data_factory/annotate/labels.py +399 -0
  8. data_factory/annotate/loop.py +359 -0
  9. data_factory/annotate/projects.py +207 -0
  10. data_factory/annotate/review.py +471 -0
  11. data_factory/annotate/schema.sql +171 -0
  12. data_factory/annotate/shapes.py +488 -0
  13. data_factory/annotate/store.py +292 -0
  14. data_factory/annotate/tools/__init__.py +39 -0
  15. data_factory/annotate/tools/fake.py +170 -0
  16. data_factory/annotate/tools/install.py +254 -0
  17. data_factory/annotate/tools/local.py +389 -0
  18. data_factory/annotate/tools/postprocess.py +175 -0
  19. data_factory/annotate/tools/protocol.py +238 -0
  20. data_factory/annotate/tools/registry.py +196 -0
  21. data_factory/annotate/tools/results.py +83 -0
  22. data_factory/annotate/tools/serve.py +106 -0
  23. data_factory/cli/__init__.py +21 -0
  24. data_factory/cli/main.py +837 -0
  25. data_factory/cli/menu.py +173 -0
  26. data_factory/cli/render.py +158 -0
  27. data_factory/core/__init__.py +11 -0
  28. data_factory/core/annotations/__init__.py +5 -0
  29. data_factory/core/annotations/coco.py +394 -0
  30. data_factory/core/annotations/csvfmt.py +174 -0
  31. data_factory/core/annotations/dota.py +98 -0
  32. data_factory/core/annotations/formats.py +626 -0
  33. data_factory/core/annotations/labelme.py +153 -0
  34. data_factory/core/annotations/model.py +334 -0
  35. data_factory/core/annotations/voc.py +231 -0
  36. data_factory/core/annotations/yolo.py +283 -0
  37. data_factory/core/bktree.py +89 -0
  38. data_factory/core/fs.py +245 -0
  39. data_factory/core/imaging.py +626 -0
  40. data_factory/core/ops/__init__.py +44 -0
  41. data_factory/core/ops/analyze.py +383 -0
  42. data_factory/core/ops/annotate_export.py +326 -0
  43. data_factory/core/ops/audit.py +274 -0
  44. data_factory/core/ops/augment.py +454 -0
  45. data_factory/core/ops/collect.py +241 -0
  46. data_factory/core/ops/combine.py +582 -0
  47. data_factory/core/ops/convert.py +212 -0
  48. data_factory/core/ops/dataset.py +246 -0
  49. data_factory/core/ops/dedup.py +235 -0
  50. data_factory/core/ops/export.py +168 -0
  51. data_factory/core/ops/image.py +407 -0
  52. data_factory/core/ops/labels.py +215 -0
  53. data_factory/core/ops/pairing.py +133 -0
  54. data_factory/core/ops/rename.py +154 -0
  55. data_factory/core/ops/report.py +323 -0
  56. data_factory/core/ops/visualize.py +164 -0
  57. data_factory/core/pairs.py +144 -0
  58. data_factory/core/parallel.py +191 -0
  59. data_factory/core/provenance.py +136 -0
  60. data_factory/core/registry.py +339 -0
  61. data_factory/core/stats.py +403 -0
  62. data_factory/core/types.py +148 -0
  63. data_factory/core/writer.py +194 -0
  64. data_factory/desktop.py +207 -0
  65. data_factory/hardware.py +243 -0
  66. data_factory/pipeline.py +192 -0
  67. data_factory/py.typed +0 -0
  68. data_factory/service/__init__.py +26 -0
  69. data_factory/service/app.py +483 -0
  70. data_factory/service/binding.py +174 -0
  71. data_factory/service/deps.py +133 -0
  72. data_factory/service/jobs.py +177 -0
  73. data_factory/service/limits.py +71 -0
  74. data_factory/service/routers/__init__.py +11 -0
  75. data_factory/service/routers/annotate.py +582 -0
  76. data_factory/service/routers/local.py +464 -0
  77. data_factory/service/routers/media.py +154 -0
  78. data_factory/service/routers/predictions.py +116 -0
  79. data_factory/service/routers/promote.py +182 -0
  80. data_factory/service/routers/tools.py +308 -0
  81. data_factory/service/static/annotate/api.js +246 -0
  82. data_factory/service/static/annotate/app.js +1696 -0
  83. data_factory/service/static/annotate/assist.js +106 -0
  84. data_factory/service/static/annotate/filmstrip.js +286 -0
  85. data_factory/service/static/annotate/hit.js +146 -0
  86. data_factory/service/static/annotate/labels.js +197 -0
  87. data_factory/service/static/annotate/render.js +286 -0
  88. data_factory/service/static/annotate/scene.js +277 -0
  89. data_factory/service/static/annotate/shapelist.js +72 -0
  90. data_factory/service/static/annotate/tools/box.js +96 -0
  91. data_factory/service/static/annotate/tools/line.js +96 -0
  92. data_factory/service/static/annotate/tools/obb.js +88 -0
  93. data_factory/service/static/annotate/tools/point.js +49 -0
  94. data_factory/service/static/annotate/tools/polygon.js +110 -0
  95. data_factory/service/static/annotate/tools/select.js +227 -0
  96. data_factory/service/static/annotate/undo.js +176 -0
  97. data_factory/service/static/annotate/viewport.js +156 -0
  98. data_factory/service/static/annotate.html +390 -0
  99. data_factory/service/static/index.html +632 -0
  100. data_factory/service/workspace.py +244 -0
  101. easyannotate/__init__.py +64 -0
  102. easyannotate/__main__.py +5 -0
  103. easyannotate/project.py +181 -0
  104. easyannotate/py.typed +0 -0
  105. easyannotate-0.4.0.dist-info/METADATA +767 -0
  106. easyannotate-0.4.0.dist-info/RECORD +109 -0
  107. easyannotate-0.4.0.dist-info/WHEEL +4 -0
  108. easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
  109. easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,399 @@
1
+ """The ordered label table -- and the correctness bug it exists to fix.
2
+
3
+ A YOLO label file stores a class *index*, never a name. The exporter today
4
+ builds that index table from ``sorted(found)`` (``core/annotations/formats.py``,
5
+ ``Dataset.labels``), so the mapping is a function of *which class names happen
6
+ to be present at export time*. Add one class mid-project and every index at or
7
+ after it shifts by one:
8
+
9
+ Monday classes = [Closed, Open] a file reading `1` means Open
10
+ Friday classes = [Closed, Handle, Open] the same file now means Handle
11
+
12
+ Nothing errors. The ``.txt`` files exported on Monday are still on disk, still
13
+ readable, and now silently mean something else. A model trained on the mixture
14
+ learns nonsense, and the only symptom is that accuracy is worse than it should
15
+ be. That is why this table is the first thing built in Phase 1.
16
+
17
+ Three rules make the index trustworthy:
18
+
19
+ * **Allocation is monotonic.** Indices come from a per-project counter, not
20
+ from ``max(idx) + 1``, so retiring the newest label and adding another does
21
+ not hand out the retired index.
22
+ * **Renaming does not renumber.** A label is an index with a name attached, not
23
+ a name with an index derived from it. Fixing a typo is safe.
24
+ * **Retiring does not compact.** A retired label keeps its index forever and
25
+ still occupies its slot in the exported class list, so the indices around it
26
+ do not move.
27
+
28
+ The counterpart to this module is :func:`class_list`, which is what an export
29
+ must use instead of sorting the names it happens to find.
30
+ """
31
+
32
+ from __future__ import annotations
33
+
34
+ import time
35
+ from dataclasses import dataclass
36
+ from typing import Any
37
+
38
+ from .projects import Project, touch
39
+ from .store import NotFound, Store, StoreError
40
+
41
+ #: Distinct, readable on both light and dark canvas, and safe against the two
42
+ #: common colour-vision deficiencies at the top of the list.
43
+ #:
44
+ #: These are ``#rrggbb``. Do **not** reach for ``core/ops/visualize.PALETTE``
45
+ #: here: that one is **BGR**, because OpenCV, and using it in CSS swaps red and
46
+ #: blue -- a bug that looks like a design choice and so survives review.
47
+ PALETTE = (
48
+ "#e6194b", "#3cb44b", "#4363d8", "#f58231", "#911eb4",
49
+ "#42d4f4", "#f032e6", "#bfef45", "#fabed4", "#469990",
50
+ "#dcbeff", "#9a6324", "#fffac8", "#800000", "#aaffc3",
51
+ "#808000", "#ffd8b1", "#000075", "#a9a9a9", "#ffe119",
52
+ )
53
+
54
+ #: Long enough for a sentence-ish class name, short enough that a paste
55
+ #: accident cannot put a kilobyte in the palette.
56
+ MAX_NAME = 120
57
+
58
+
59
+ @dataclass(slots=True)
60
+ class Label:
61
+ project_id: str
62
+ idx: int
63
+ name: str
64
+ colour: str
65
+ created_at: float
66
+ retired_at: float | None = None
67
+
68
+ @property
69
+ def is_live(self) -> bool:
70
+ return self.retired_at is None
71
+
72
+ def as_dict(self) -> dict[str, Any]:
73
+ return {
74
+ "idx": self.idx,
75
+ "name": self.name,
76
+ "colour": self.colour,
77
+ "retired": not self.is_live,
78
+ }
79
+
80
+
81
+ def _row(row: Any) -> Label:
82
+ return Label(
83
+ project_id=row["project_id"],
84
+ idx=row["idx"],
85
+ name=row["name"],
86
+ colour=row["colour"],
87
+ created_at=row["created_at"],
88
+ retired_at=row["retired_at"],
89
+ )
90
+
91
+
92
+ def clean_name(name: str) -> str:
93
+ """Normalise a name typed into the canvas.
94
+
95
+ Whitespace is collapsed rather than merely stripped, because "front door"
96
+ and "front door" arriving as two classes is a real and very annoying way
97
+ to split a dataset in half.
98
+ """
99
+ text = " ".join((name or "").split())
100
+ if not text:
101
+ raise StoreError("a label needs a name")
102
+ if len(text) > MAX_NAME:
103
+ raise StoreError(f"label names are limited to {MAX_NAME} characters")
104
+ return text
105
+
106
+
107
+ def colour_for(idx: int) -> str:
108
+ return PALETTE[idx % len(PALETTE)]
109
+
110
+
111
+ def listing(store: Store, project: Project, *, include_retired: bool = False) -> list[Label]:
112
+ """Every label, in index order. Index order *is* the class order."""
113
+ sql = "SELECT * FROM label WHERE project_id = ?"
114
+ if not include_retired:
115
+ sql += " AND retired_at IS NULL"
116
+ sql += " ORDER BY idx"
117
+ return [_row(r) for r in store.query(sql, (project.id,))]
118
+
119
+
120
+ def get(store: Store, project: Project, idx: int) -> Label:
121
+ row = store.one(
122
+ "SELECT * FROM label WHERE project_id = ? AND idx = ?", (project.id, int(idx))
123
+ )
124
+ if row is None:
125
+ raise NotFound(f"no label with index {idx}")
126
+ return _row(row)
127
+
128
+
129
+ def by_name(store: Store, project: Project, name: str) -> Label | None:
130
+ """Find a *live* label by name. Retired names do not match."""
131
+ row = store.one(
132
+ "SELECT * FROM label WHERE project_id = ? AND name = ? AND retired_at IS NULL",
133
+ (project.id, clean_name(name)),
134
+ )
135
+ return _row(row) if row else None
136
+
137
+
138
+ def ensure(store: Store, project: Project, name: str, *, colour: str = "") -> Label:
139
+ """Return the label with this name, creating it if it is new.
140
+
141
+ This is the "draw a box, type a name, press Enter" path, and it is the
142
+ reason there is no schema step in this product. It is idempotent, so the
143
+ editor can call it on every save without checking first.
144
+ """
145
+ clean = clean_name(name)
146
+ existing = by_name(store, project, clean)
147
+ if existing is not None:
148
+ return existing
149
+
150
+ now = time.time()
151
+ with store.write() as conn:
152
+ # Re-read the counter inside the transaction. Two browser tabs both
153
+ # creating a label at once would otherwise both see the same
154
+ # next_label_idx and collide on the primary key.
155
+ row = conn.execute(
156
+ "SELECT next_label_idx FROM project WHERE id = ?", (project.id,)
157
+ ).fetchone()
158
+ if row is None:
159
+ raise NotFound("no such project")
160
+ idx = int(row["next_label_idx"])
161
+ conn.execute(
162
+ "INSERT INTO label (project_id, idx, name, colour, created_at)"
163
+ " VALUES (?, ?, ?, ?, ?)",
164
+ (project.id, idx, clean, colour or colour_for(idx), now),
165
+ )
166
+ conn.execute(
167
+ "UPDATE project SET next_label_idx = ? WHERE id = ?", (idx + 1, project.id)
168
+ )
169
+ touch(conn, project.id)
170
+
171
+ project.next_label_idx = idx + 1
172
+ return Label(
173
+ project_id=project.id,
174
+ idx=idx,
175
+ name=clean,
176
+ colour=colour or colour_for(idx),
177
+ created_at=now,
178
+ )
179
+
180
+
181
+ def rename(store: Store, project: Project, idx: int, name: str) -> Label:
182
+ """Change a label's name. **The index does not move.**
183
+
184
+ Every ``.txt`` already exported keeps meaning what it meant; only the
185
+ human-readable name in ``data.yaml`` changes. Fixing "Cosed" to "Closed"
186
+ must never be a data-corrupting act.
187
+ """
188
+ clean = clean_name(name)
189
+ label = get(store, project, idx)
190
+ clash = by_name(store, project, clean)
191
+ if clash is not None and clash.idx != label.idx:
192
+ raise StoreError(f"another label is already called {clean!r}")
193
+ with store.write() as conn:
194
+ conn.execute(
195
+ "UPDATE label SET name = ? WHERE project_id = ? AND idx = ?",
196
+ (clean, project.id, int(idx)),
197
+ )
198
+ touch(conn, project.id)
199
+ label.name = clean
200
+ return label
201
+
202
+
203
+ def recolour(store: Store, project: Project, idx: int, colour: str) -> Label:
204
+ text = (colour or "").strip().lower()
205
+ if not (len(text) == 7 and text[0] == "#" and all(c in "0123456789abcdef" for c in text[1:])):
206
+ raise StoreError("colour must look like #rrggbb")
207
+ label = get(store, project, idx)
208
+ with store.write() as conn:
209
+ conn.execute(
210
+ "UPDATE label SET colour = ? WHERE project_id = ? AND idx = ?",
211
+ (text, project.id, int(idx)),
212
+ )
213
+ touch(conn, project.id)
214
+ label.colour = text
215
+ return label
216
+
217
+
218
+ def retire(store: Store, project: Project, idx: int) -> Label:
219
+ """Hide a label from the palette without deleting it.
220
+
221
+ Not a DELETE, for two reasons. Shapes already drawn with it would lose
222
+ their referent, and -- the quiet one -- removing the row would leave a hole
223
+ that a future ``class_list`` might close up, moving every index after it.
224
+ """
225
+ label = get(store, project, idx)
226
+ if not label.is_live:
227
+ return label
228
+ now = time.time()
229
+ with store.write() as conn:
230
+ conn.execute(
231
+ "UPDATE label SET retired_at = ? WHERE project_id = ? AND idx = ?",
232
+ (now, project.id, int(idx)),
233
+ )
234
+ touch(conn, project.id)
235
+ label.retired_at = now
236
+ return label
237
+
238
+
239
+ def restore(store: Store, project: Project, idx: int) -> Label:
240
+ """Un-retire a label, back into its original index."""
241
+ label = get(store, project, idx)
242
+ if label.is_live:
243
+ return label
244
+ clash = by_name(store, project, label.name)
245
+ if clash is not None:
246
+ raise StoreError(
247
+ f"{label.name!r} is in use by index {clash.idx}; rename that one first"
248
+ )
249
+ with store.write() as conn:
250
+ conn.execute(
251
+ "UPDATE label SET retired_at = NULL WHERE project_id = ? AND idx = ?",
252
+ (project.id, int(idx)),
253
+ )
254
+ touch(conn, project.id)
255
+ label.retired_at = None
256
+ return label
257
+
258
+
259
+ def variants(store: Store, project: Project) -> dict[str, list[Label]]:
260
+ """Live labels that differ only by case or spacing, grouped.
261
+
262
+ The real door dataset carries exactly this defect -- ``Close``, ``Open``,
263
+ ``open`` and ``close`` as four separate classes -- because two people
264
+ annotated it in different tools. Importing them faithfully is right;
265
+ inventing a merge on the way in would silently change somebody's data. But
266
+ the user has to be *told*, or they will train a four-class model by
267
+ accident and never know why it is confused.
268
+ """
269
+ grouped: dict[str, list[Label]] = {}
270
+ for label in listing(store, project):
271
+ grouped.setdefault(label.name.strip().lower(), []).append(label)
272
+ return {key: found for key, found in grouped.items() if len(found) > 1}
273
+
274
+
275
+ def merge(store: Store, project: Project, source_idx: int, target_idx: int) -> int:
276
+ """Move every shape from one class onto another and retire the source.
277
+
278
+ Returns how many shapes moved.
279
+
280
+ This is the only safe way to fold ``close`` into ``Close``. Doing it by
281
+ renaming would either be refused (the name is taken) or, if it were
282
+ allowed, leave two live labels with the same name and no way to tell which
283
+ index an export should use. Here the source index is *retired*, so it keeps
284
+ its slot in the class list and nothing after it shifts -- the merged class
285
+ simply has no objects any more.
286
+ """
287
+ source = get(store, project, source_idx)
288
+ target = get(store, project, target_idx)
289
+ if source.idx == target.idx:
290
+ raise StoreError("a class cannot be merged into itself")
291
+ if not target.is_live:
292
+ raise StoreError(f"{target.name!r} is retired; restore it before merging into it")
293
+
294
+ now = time.time()
295
+ affected = [
296
+ row["image_id"]
297
+ for row in store.query(
298
+ "SELECT DISTINCT image_id FROM shape WHERE project_id = ? AND label_idx = ?",
299
+ (project.id, source.idx),
300
+ )
301
+ ]
302
+ with store.write() as conn:
303
+ moved = conn.execute(
304
+ "UPDATE shape SET label_idx = ? WHERE project_id = ? AND label_idx = ?",
305
+ (target.idx, project.id, source.idx),
306
+ ).rowcount
307
+ conn.execute(
308
+ "UPDATE label SET retired_at = ? WHERE project_id = ? AND idx = ?",
309
+ (now, project.id, source.idx),
310
+ )
311
+ touch(conn, project.id)
312
+ # Two predictions that were "Close" and "close" were a class conflict;
313
+ # merged, they are not. The review order has to know.
314
+ from . import images as images_mod
315
+ from . import shapes as shapes_mod
316
+
317
+ for image_id in affected:
318
+ shapes_mod.refresh_uncertainty(store, project, images_mod.get(store, project, image_id))
319
+ return int(moved)
320
+
321
+
322
+ def class_list(store: Store, project: Project) -> list[str]:
323
+ """The export class list: names positioned at their own indices.
324
+
325
+ This is the function an export calls instead of sorting whatever names it
326
+ finds. Two properties matter:
327
+
328
+ * Retired labels still occupy their slot, so nothing after them shifts.
329
+ * The list is dense from 0, because YOLO addresses it positionally -- a gap
330
+ would make every later index wrong, which is the very bug this replaces.
331
+
332
+ A gap can only appear in a database edited by hand, but filling it costs
333
+ one branch and turns a silent mislabelling into a visible placeholder.
334
+ """
335
+ names = export_names(store, project)
336
+ if not names:
337
+ return []
338
+ highest = max(names)
339
+ return [names.get(i, f"_unused_{i}") for i in range(highest + 1)]
340
+
341
+
342
+ def export_names(store: Store, project: Project) -> dict[int, str]:
343
+ """Index -> the name written into exported files. **Unique, always.**
344
+
345
+ Names are unique among live labels only, so a retired class can share its
346
+ name with a live one -- by renaming, or by reusing the name, which is
347
+ allowed on purpose. Writers map shapes to indices *by name*, so a shared
348
+ name silently merged the retired class's shapes into the live one. Live
349
+ labels keep their names; a retired label whose name is already taken is
350
+ written as ``name__retired_<idx>``, visibly and reversibly distinct.
351
+ """
352
+ rows = store.query(
353
+ "SELECT idx, name, retired_at FROM label WHERE project_id = ?"
354
+ " ORDER BY (retired_at IS NOT NULL), idx",
355
+ (project.id,),
356
+ )
357
+ out: dict[int, str] = {}
358
+ taken: set[str] = set()
359
+ for row in rows:
360
+ idx, name = int(row["idx"]), str(row["name"])
361
+ if name in taken:
362
+ name = f"{name}__retired_{idx}"
363
+ taken.add(name)
364
+ out[idx] = name
365
+ return out
366
+
367
+
368
+ def adopt(store: Store, project: Project, names: list[str]) -> list[Label]:
369
+ """Take on an existing class order wholesale, preserving positions.
370
+
371
+ Used when a project is started from a dataset that already has a
372
+ ``data.yaml``: those indices are already baked into label files on disk, so
373
+ we inherit them rather than re-deriving an order of our own and
374
+ invalidating every one of them.
375
+ """
376
+ created: list[Label] = []
377
+ now = time.time()
378
+ with store.write() as conn:
379
+ row = conn.execute(
380
+ "SELECT next_label_idx FROM project WHERE id = ?", (project.id,)
381
+ ).fetchone()
382
+ if row is None:
383
+ raise NotFound("no such project")
384
+ if int(row["next_label_idx"]) != 0:
385
+ raise StoreError("a class order can only be adopted before any label exists")
386
+ for idx, raw in enumerate(names):
387
+ clean = clean_name(raw)
388
+ conn.execute(
389
+ "INSERT INTO label (project_id, idx, name, colour, created_at)"
390
+ " VALUES (?, ?, ?, ?, ?)",
391
+ (project.id, idx, clean, colour_for(idx), now),
392
+ )
393
+ created.append(Label(project.id, idx, clean, colour_for(idx), now))
394
+ conn.execute(
395
+ "UPDATE project SET next_label_idx = ? WHERE id = ?", (len(names), project.id)
396
+ )
397
+ touch(conn, project.id)
398
+ project.next_label_idx = len(names)
399
+ return created