easyannotate 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. data_factory/__init__.py +82 -0
  2. data_factory/__main__.py +6 -0
  3. data_factory/annotate/__init__.py +15 -0
  4. data_factory/annotate/attach.py +398 -0
  5. data_factory/annotate/demo.py +161 -0
  6. data_factory/annotate/images.py +443 -0
  7. data_factory/annotate/labels.py +399 -0
  8. data_factory/annotate/loop.py +359 -0
  9. data_factory/annotate/projects.py +207 -0
  10. data_factory/annotate/review.py +471 -0
  11. data_factory/annotate/schema.sql +171 -0
  12. data_factory/annotate/shapes.py +488 -0
  13. data_factory/annotate/store.py +292 -0
  14. data_factory/annotate/tools/__init__.py +39 -0
  15. data_factory/annotate/tools/fake.py +170 -0
  16. data_factory/annotate/tools/install.py +254 -0
  17. data_factory/annotate/tools/local.py +389 -0
  18. data_factory/annotate/tools/postprocess.py +175 -0
  19. data_factory/annotate/tools/protocol.py +238 -0
  20. data_factory/annotate/tools/registry.py +196 -0
  21. data_factory/annotate/tools/results.py +83 -0
  22. data_factory/annotate/tools/serve.py +106 -0
  23. data_factory/cli/__init__.py +21 -0
  24. data_factory/cli/main.py +837 -0
  25. data_factory/cli/menu.py +173 -0
  26. data_factory/cli/render.py +158 -0
  27. data_factory/core/__init__.py +11 -0
  28. data_factory/core/annotations/__init__.py +5 -0
  29. data_factory/core/annotations/coco.py +394 -0
  30. data_factory/core/annotations/csvfmt.py +174 -0
  31. data_factory/core/annotations/dota.py +98 -0
  32. data_factory/core/annotations/formats.py +626 -0
  33. data_factory/core/annotations/labelme.py +153 -0
  34. data_factory/core/annotations/model.py +334 -0
  35. data_factory/core/annotations/voc.py +231 -0
  36. data_factory/core/annotations/yolo.py +283 -0
  37. data_factory/core/bktree.py +89 -0
  38. data_factory/core/fs.py +245 -0
  39. data_factory/core/imaging.py +626 -0
  40. data_factory/core/ops/__init__.py +44 -0
  41. data_factory/core/ops/analyze.py +383 -0
  42. data_factory/core/ops/annotate_export.py +326 -0
  43. data_factory/core/ops/audit.py +274 -0
  44. data_factory/core/ops/augment.py +454 -0
  45. data_factory/core/ops/collect.py +241 -0
  46. data_factory/core/ops/combine.py +582 -0
  47. data_factory/core/ops/convert.py +212 -0
  48. data_factory/core/ops/dataset.py +246 -0
  49. data_factory/core/ops/dedup.py +235 -0
  50. data_factory/core/ops/export.py +168 -0
  51. data_factory/core/ops/image.py +407 -0
  52. data_factory/core/ops/labels.py +215 -0
  53. data_factory/core/ops/pairing.py +133 -0
  54. data_factory/core/ops/rename.py +154 -0
  55. data_factory/core/ops/report.py +323 -0
  56. data_factory/core/ops/visualize.py +164 -0
  57. data_factory/core/pairs.py +144 -0
  58. data_factory/core/parallel.py +191 -0
  59. data_factory/core/provenance.py +136 -0
  60. data_factory/core/registry.py +339 -0
  61. data_factory/core/stats.py +403 -0
  62. data_factory/core/types.py +148 -0
  63. data_factory/core/writer.py +194 -0
  64. data_factory/desktop.py +207 -0
  65. data_factory/hardware.py +243 -0
  66. data_factory/pipeline.py +192 -0
  67. data_factory/py.typed +0 -0
  68. data_factory/service/__init__.py +26 -0
  69. data_factory/service/app.py +483 -0
  70. data_factory/service/binding.py +174 -0
  71. data_factory/service/deps.py +133 -0
  72. data_factory/service/jobs.py +177 -0
  73. data_factory/service/limits.py +71 -0
  74. data_factory/service/routers/__init__.py +11 -0
  75. data_factory/service/routers/annotate.py +582 -0
  76. data_factory/service/routers/local.py +464 -0
  77. data_factory/service/routers/media.py +154 -0
  78. data_factory/service/routers/predictions.py +116 -0
  79. data_factory/service/routers/promote.py +182 -0
  80. data_factory/service/routers/tools.py +308 -0
  81. data_factory/service/static/annotate/api.js +246 -0
  82. data_factory/service/static/annotate/app.js +1696 -0
  83. data_factory/service/static/annotate/assist.js +106 -0
  84. data_factory/service/static/annotate/filmstrip.js +286 -0
  85. data_factory/service/static/annotate/hit.js +146 -0
  86. data_factory/service/static/annotate/labels.js +197 -0
  87. data_factory/service/static/annotate/render.js +286 -0
  88. data_factory/service/static/annotate/scene.js +277 -0
  89. data_factory/service/static/annotate/shapelist.js +72 -0
  90. data_factory/service/static/annotate/tools/box.js +96 -0
  91. data_factory/service/static/annotate/tools/line.js +96 -0
  92. data_factory/service/static/annotate/tools/obb.js +88 -0
  93. data_factory/service/static/annotate/tools/point.js +49 -0
  94. data_factory/service/static/annotate/tools/polygon.js +110 -0
  95. data_factory/service/static/annotate/tools/select.js +227 -0
  96. data_factory/service/static/annotate/undo.js +176 -0
  97. data_factory/service/static/annotate/viewport.js +156 -0
  98. data_factory/service/static/annotate.html +390 -0
  99. data_factory/service/static/index.html +632 -0
  100. data_factory/service/workspace.py +244 -0
  101. easyannotate/__init__.py +64 -0
  102. easyannotate/__main__.py +5 -0
  103. easyannotate/project.py +181 -0
  104. easyannotate/py.typed +0 -0
  105. easyannotate-0.4.0.dist-info/METADATA +767 -0
  106. easyannotate-0.4.0.dist-info/RECORD +109 -0
  107. easyannotate-0.4.0.dist-info/WHEEL +4 -0
  108. easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
  109. easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,443 @@
1
+ """Images within a project: the list, the ordering, and the resume point.
2
+
3
+ The editor asks this module three questions constantly -- what images are
4
+ there, which one comes next, and how much is left -- so all three are indexed
5
+ queries rather than directory walks. That is the reason the image list is
6
+ database-backed at all: ``fs.iter_images`` over 20,000 files is fine once at
7
+ import and hopeless as a per-request operation.
8
+
9
+ Paths stored here are POSIX-relative to the project root, never absolute. A row
10
+ holding an absolute path is a row that could eventually be joined onto
11
+ something a client influences, and it makes a database written on Windows
12
+ useless on Linux.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import time
18
+ from collections.abc import Iterable
19
+ from dataclasses import dataclass
20
+ from pathlib import Path, PurePosixPath
21
+ from typing import Any, TypeVar
22
+
23
+ from .projects import Project, touch
24
+ from .store import NotFound, Store, StoreError, new_id, require_id
25
+
26
+ T = TypeVar("T")
27
+
28
+ #: todo nothing has happened to it yet
29
+ #: predicted a model wrote shapes onto it, nobody has looked
30
+ #: in_progress someone has started
31
+ #: reviewed a person has signed it off -- the only state export trusts
32
+ #: skipped an explicit "I cannot tell", which must never become training data
33
+ STATES = ("todo", "predicted", "in_progress", "reviewed", "skipped")
34
+
35
+ #: States that represent finished human judgement.
36
+ DONE_STATES = ("reviewed",)
37
+
38
+
39
+ @dataclass(slots=True)
40
+ class Image:
41
+ id: str
42
+ project_id: str
43
+ rel_path: str
44
+ width: int
45
+ height: int
46
+ seq: int
47
+ state: str
48
+ version: int
49
+ created_at: float
50
+ updated_at: float
51
+ uncertainty: float = 0.0
52
+
53
+ @property
54
+ def name(self) -> str:
55
+ return PurePosixPath(self.rel_path).name
56
+
57
+ @property
58
+ def stem(self) -> str:
59
+ return PurePosixPath(self.rel_path).stem
60
+
61
+ def as_dict(self) -> dict[str, Any]:
62
+ return {
63
+ "id": self.id,
64
+ "path": self.rel_path,
65
+ "name": self.name,
66
+ "width": self.width,
67
+ "height": self.height,
68
+ "seq": self.seq,
69
+ "state": self.state,
70
+ "version": self.version,
71
+ "uncertainty": round(self.uncertainty, 3),
72
+ }
73
+
74
+
75
+ #: Ways a listing can be ordered. "uncertain" puts the predictions most likely
76
+ #: to be wrong first -- review time spent where the model needs it.
77
+ ORDERS = ("", "uncertain")
78
+
79
+
80
+ def _order_by(order: str) -> str:
81
+ if order not in ORDERS:
82
+ raise StoreError(f"order must be one of: {', '.join(o or '(file order)' for o in ORDERS)}")
83
+ return " ORDER BY uncertainty DESC, seq" if order == "uncertain" else " ORDER BY seq"
84
+
85
+
86
+ def _row(row: Any) -> Image:
87
+ return Image(
88
+ id=row["id"],
89
+ project_id=row["project_id"],
90
+ rel_path=row["rel_path"],
91
+ width=row["width"],
92
+ height=row["height"],
93
+ seq=row["seq"],
94
+ state=row["state"],
95
+ version=row["version"],
96
+ created_at=row["created_at"],
97
+ updated_at=row["updated_at"],
98
+ uncertainty=float(row["uncertainty"]) if "uncertainty" in row.keys() else 0.0,
99
+ )
100
+
101
+
102
+ def clean_rel_path(raw: str) -> str:
103
+ """Normalise a relative path, refusing anything that could escape.
104
+
105
+ Separators are normalised because a Windows scan and a Linux scan of the
106
+ same folder should produce the same row. Escapes are *refused* rather than
107
+ stripped -- consistent with the service layer, where quietly sanitising a
108
+ hostile path mostly teaches the sender what to try next.
109
+ """
110
+ text = (raw or "").strip().replace("\\", "/")
111
+ if not text:
112
+ raise StoreError("an image needs a path")
113
+ if text.startswith("/") or (len(text) > 1 and text[1] == ":"):
114
+ raise StoreError(f"image paths must be relative, got {raw!r}")
115
+ parts = PurePosixPath(text).parts
116
+ if any(part == ".." for part in parts):
117
+ raise StoreError(f"image path escapes the project: {raw!r}")
118
+ kept = [p for p in parts if p != "."]
119
+ if not kept:
120
+ raise StoreError("an image needs a path")
121
+ return PurePosixPath(*kept).as_posix()
122
+
123
+
124
+ def _check_state(state: str) -> str:
125
+ if state not in STATES:
126
+ raise StoreError(f"state must be one of {', '.join(STATES)}")
127
+ return state
128
+
129
+
130
+ def add(
131
+ store: Store,
132
+ project: Project,
133
+ rel_path: str,
134
+ *,
135
+ width: int = 0,
136
+ height: int = 0,
137
+ state: str = "todo",
138
+ ) -> Image:
139
+ """Register one image. Idempotent on path, so a re-scan is harmless."""
140
+ return add_many(store, project, [(rel_path, width, height)], state=state)[0]
141
+
142
+
143
+ def add_many(
144
+ store: Store,
145
+ project: Project,
146
+ entries: Iterable[tuple[str, int, int]],
147
+ *,
148
+ state: str = "todo",
149
+ ) -> list[Image]:
150
+ """Bulk register. One transaction, because 20,000 commits is a minute.
151
+
152
+ ``seq`` is assigned by insertion order and continues from whatever is
153
+ already there, which keeps a second import appended rather than shuffled
154
+ into the middle of someone's half-finished pass.
155
+ """
156
+ _check_state(state)
157
+ cleaned = [(clean_rel_path(p), int(w), int(h)) for p, w, h in entries]
158
+ now = time.time()
159
+ made: list[Image] = []
160
+
161
+ with store.write() as conn:
162
+ row = conn.execute(
163
+ "SELECT COALESCE(MAX(seq), -1) AS top FROM image WHERE project_id = ?",
164
+ (project.id,),
165
+ ).fetchone()
166
+ seq = int(row["top"]) + 1
167
+
168
+ for rel_path, width, height in cleaned:
169
+ existing = conn.execute(
170
+ "SELECT * FROM image WHERE project_id = ? AND rel_path = ?",
171
+ (project.id, rel_path),
172
+ ).fetchone()
173
+ if existing is not None:
174
+ # A re-scan may have learned the real dimensions; take those,
175
+ # but never touch state or version -- that is someone's work.
176
+ if width and height and (
177
+ existing["width"] != width or existing["height"] != height
178
+ ):
179
+ conn.execute(
180
+ "UPDATE image SET width = ?, height = ? WHERE id = ?",
181
+ (width, height, existing["id"]),
182
+ )
183
+ existing = conn.execute(
184
+ "SELECT * FROM image WHERE id = ?", (existing["id"],)
185
+ ).fetchone()
186
+ made.append(_row(existing))
187
+ continue
188
+
189
+ image_id = new_id()
190
+ conn.execute(
191
+ "INSERT INTO image (id, project_id, rel_path, width, height, seq,"
192
+ " state, version, created_at, updated_at)"
193
+ " VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?)",
194
+ (image_id, project.id, rel_path, width, height, seq, state, now, now),
195
+ )
196
+ made.append(
197
+ Image(image_id, project.id, rel_path, width, height, seq, state, 0, now, now)
198
+ )
199
+ seq += 1
200
+ touch(conn, project.id)
201
+
202
+ return made
203
+
204
+
205
+ def get(store: Store, project: Project, image_id: str) -> Image:
206
+ require_id(image_id, "image id")
207
+ row = store.one(
208
+ "SELECT * FROM image WHERE id = ? AND project_id = ?", (image_id, project.id)
209
+ )
210
+ if row is None:
211
+ raise NotFound("no such image")
212
+ return _row(row)
213
+
214
+
215
+ def by_path(store: Store, project: Project, rel_path: str) -> Image | None:
216
+ row = store.one(
217
+ "SELECT * FROM image WHERE project_id = ? AND rel_path = ?",
218
+ (project.id, clean_rel_path(rel_path)),
219
+ )
220
+ return _row(row) if row else None
221
+
222
+
223
+ def listing_all(store: Store, project: Project) -> list[Image]:
224
+ """Every image in the project, in order. For import-time matching."""
225
+ return [
226
+ _row(row)
227
+ for row in store.query(
228
+ "SELECT * FROM image WHERE project_id = ? ORDER BY seq", (project.id,)
229
+ )
230
+ ]
231
+
232
+
233
+ class Matcher:
234
+ """Which project image does a label file belong to?
235
+
236
+ By path first, and by bare filename stem only when that stem settles it.
237
+ ``train/001.jpg`` and ``val/001.jpg`` share a stem; matching on it alone
238
+ gave both images whichever label came first, and in attached mode the
239
+ next save wrote that image's boxes into the other one's file on disk.
240
+
241
+ When several images share the stem, the one sharing the most trailing
242
+ folders wins -- so ``labels/val/001.txt`` still finds
243
+ ``images/val/001.jpg`` -- and a tie is *ambiguous*: recorded in
244
+ :attr:`ambiguous` for the caller to report, never guessed.
245
+ """
246
+
247
+ def __init__(self, images: Iterable[Image]) -> None:
248
+ self._by_stem: dict[str, list[Image]] = {}
249
+ for image in images:
250
+ self._by_stem.setdefault(image.stem, []).append(image)
251
+ self.ambiguous: set[str] = set()
252
+
253
+ def find(self, name: str) -> Image | None:
254
+ """``name`` is a bare filename or a relative path, extension optional."""
255
+ parts = _match_parts(name)
256
+ if not parts:
257
+ return None
258
+ candidates = self._by_stem.get(parts[-1], [])
259
+ if len(candidates) <= 1:
260
+ return candidates[0] if candidates else None
261
+ scored = sorted(
262
+ ((_shared_tail(parts, _match_parts(c.rel_path)), c) for c in candidates),
263
+ key=lambda pair: pair[0],
264
+ reverse=True,
265
+ )
266
+ best, runner_up = scored[0][0], scored[1][0]
267
+ if best > 1 and best > runner_up:
268
+ return scored[0][1]
269
+ self.ambiguous.add(name)
270
+ return None
271
+
272
+
273
+ def assign(images: Iterable[Image], named: Iterable[tuple[str, T]]) -> tuple[dict[str, T], list[str]]:
274
+ """Pair each ``(name, value)`` with the image it belongs to.
275
+
276
+ Returns ``{image id: value}`` and the names that could not be placed
277
+ safely -- ambiguous, or a second claim on an image already taken. Those
278
+ are for the caller to report; placing them anyway is how one image's
279
+ labels end up on another.
280
+ """
281
+ matcher = Matcher(images)
282
+ placed: dict[str, T] = {}
283
+ unsure: list[str] = []
284
+ for name, value in named:
285
+ image = matcher.find(name)
286
+ if image is None:
287
+ if name in matcher.ambiguous:
288
+ unsure.append(name)
289
+ continue
290
+ if image.id in placed:
291
+ unsure.append(name)
292
+ continue
293
+ placed[image.id] = value
294
+ return placed, unsure
295
+
296
+
297
+ def item_name(item: Any, root: Path) -> str:
298
+ """The best name a dataset item has for matching: its image's path
299
+ relative to ``root`` when it has one, else the filename it records."""
300
+ image = getattr(item, "image", None)
301
+ if image is not None:
302
+ try:
303
+ return Path(image).resolve().relative_to(Path(root).resolve()).as_posix()
304
+ except ValueError:
305
+ pass
306
+ return item.annotation.filename
307
+
308
+
309
+ def _match_parts(name: str) -> list[str]:
310
+ path = PurePosixPath(str(name).replace("\\", "/"))
311
+ parts = [p for p in path.parts if p not in ("", ".", "/")]
312
+ if parts:
313
+ parts[-1] = PurePosixPath(parts[-1]).stem
314
+ return parts
315
+
316
+
317
+ def _shared_tail(a: list[str], b: list[str]) -> int:
318
+ count = 0
319
+ for x, y in zip(reversed(a), reversed(b)):
320
+ if x != y:
321
+ break
322
+ count += 1
323
+ return count
324
+
325
+
326
+ def listing(
327
+ store: Store,
328
+ project: Project,
329
+ *,
330
+ state: str = "",
331
+ limit: int = 500,
332
+ offset: int = 0,
333
+ order: str = "",
334
+ ) -> list[Image]:
335
+ sql = "SELECT * FROM image WHERE project_id = ?"
336
+ params: list[Any] = [project.id]
337
+ if state:
338
+ sql += " AND state = ?"
339
+ params.append(_check_state(state))
340
+ sql += _order_by(order) + " LIMIT ? OFFSET ?"
341
+ params.extend([max(1, min(int(limit), 5000)), max(0, int(offset))])
342
+ return [_row(r) for r in store.query(sql, tuple(params))]
343
+
344
+
345
+ def resume_point(store: Store, project: Project) -> Image | None:
346
+ """The image to open when a project is opened: the first unfinished one.
347
+
348
+ Resume is the default, not a feature behind a button. Closing the browser
349
+ at image 1000 and coming back to image 1 is why people abandon annotation
350
+ tools.
351
+ """
352
+ row = store.one(
353
+ "SELECT * FROM image WHERE project_id = ? AND state NOT IN ('reviewed', 'skipped')"
354
+ " ORDER BY seq LIMIT 1",
355
+ (project.id,),
356
+ )
357
+ if row is None:
358
+ # Everything is done; land on the first image rather than nowhere.
359
+ row = store.one(
360
+ "SELECT * FROM image WHERE project_id = ? ORDER BY seq LIMIT 1", (project.id,)
361
+ )
362
+ return _row(row) if row else None
363
+
364
+
365
+ def neighbour(store: Store, project: Project, image: Image, step: int = 1) -> Image | None:
366
+ """The next or previous image by sequence."""
367
+ if step >= 0:
368
+ row = store.one(
369
+ "SELECT * FROM image WHERE project_id = ? AND seq > ? ORDER BY seq LIMIT 1",
370
+ (project.id, image.seq),
371
+ )
372
+ else:
373
+ row = store.one(
374
+ "SELECT * FROM image WHERE project_id = ? AND seq < ? ORDER BY seq DESC LIMIT 1",
375
+ (project.id, image.seq),
376
+ )
377
+ return _row(row) if row else None
378
+
379
+
380
+ def set_state(store: Store, project: Project, image: Image, state: str) -> Image:
381
+ _check_state(state)
382
+ now = time.time()
383
+ with store.write() as conn:
384
+ conn.execute(
385
+ "UPDATE image SET state = ?, updated_at = ? WHERE id = ? AND project_id = ?",
386
+ (state, now, image.id, project.id),
387
+ )
388
+ touch(conn, project.id)
389
+ image.state = state
390
+ image.updated_at = now
391
+ return image
392
+
393
+
394
+ def set_size(store: Store, project: Project, image: Image, width: int, height: int) -> Image:
395
+ with store.write() as conn:
396
+ conn.execute(
397
+ "UPDATE image SET width = ?, height = ? WHERE id = ? AND project_id = ?",
398
+ (int(width), int(height), image.id, project.id),
399
+ )
400
+ image.width, image.height = int(width), int(height)
401
+ return image
402
+
403
+
404
+ def position(
405
+ store: Store, project: Project, image_id: str, *, state: str = "", order: str = ""
406
+ ) -> int | None:
407
+ """Where an image sits in the listing for ``state`` (all, if empty).
408
+
409
+ What lets the filmstrip scroll to the open image in a 20,000-image project
410
+ without fetching the 19,000 rows before it. ``None`` when the image is not
411
+ in that listing -- open, but filtered out.
412
+ """
413
+ _order_by(order) # validates
414
+ sql = "SELECT seq, state, uncertainty FROM image WHERE project_id = ? AND id = ?"
415
+ row = store.one(sql, (project.id, image_id))
416
+ if row is None or (state and row["state"] != _check_state(state)):
417
+ return None
418
+ if order == "uncertain":
419
+ count_sql = (
420
+ "SELECT COUNT(*) AS n FROM image WHERE project_id = ?"
421
+ " AND (uncertainty > ? OR (uncertainty = ? AND seq < ?))"
422
+ )
423
+ u = float(row["uncertainty"])
424
+ params: list[Any] = [project.id, u, u, int(row["seq"])]
425
+ else:
426
+ count_sql = "SELECT COUNT(*) AS n FROM image WHERE project_id = ? AND seq < ?"
427
+ params = [project.id, int(row["seq"])]
428
+ if state:
429
+ count_sql += " AND state = ?"
430
+ params.append(state)
431
+ return int(store.one(count_sql, tuple(params))["n"])
432
+
433
+
434
+ def counts(store: Store, project: Project) -> dict[str, int]:
435
+ """How much is left, by state. The number the progress bar shows."""
436
+ out = {state: 0 for state in STATES}
437
+ for row in store.query(
438
+ "SELECT state, COUNT(*) AS n FROM image WHERE project_id = ? GROUP BY state",
439
+ (project.id,),
440
+ ):
441
+ out[str(row["state"])] = int(row["n"])
442
+ out["total"] = sum(out[s] for s in STATES)
443
+ return out