easyannotate 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. data_factory/__init__.py +82 -0
  2. data_factory/__main__.py +6 -0
  3. data_factory/annotate/__init__.py +15 -0
  4. data_factory/annotate/attach.py +398 -0
  5. data_factory/annotate/demo.py +161 -0
  6. data_factory/annotate/images.py +443 -0
  7. data_factory/annotate/labels.py +399 -0
  8. data_factory/annotate/loop.py +359 -0
  9. data_factory/annotate/projects.py +207 -0
  10. data_factory/annotate/review.py +471 -0
  11. data_factory/annotate/schema.sql +171 -0
  12. data_factory/annotate/shapes.py +488 -0
  13. data_factory/annotate/store.py +292 -0
  14. data_factory/annotate/tools/__init__.py +39 -0
  15. data_factory/annotate/tools/fake.py +170 -0
  16. data_factory/annotate/tools/install.py +254 -0
  17. data_factory/annotate/tools/local.py +389 -0
  18. data_factory/annotate/tools/postprocess.py +175 -0
  19. data_factory/annotate/tools/protocol.py +238 -0
  20. data_factory/annotate/tools/registry.py +196 -0
  21. data_factory/annotate/tools/results.py +83 -0
  22. data_factory/annotate/tools/serve.py +106 -0
  23. data_factory/cli/__init__.py +21 -0
  24. data_factory/cli/main.py +837 -0
  25. data_factory/cli/menu.py +173 -0
  26. data_factory/cli/render.py +158 -0
  27. data_factory/core/__init__.py +11 -0
  28. data_factory/core/annotations/__init__.py +5 -0
  29. data_factory/core/annotations/coco.py +394 -0
  30. data_factory/core/annotations/csvfmt.py +174 -0
  31. data_factory/core/annotations/dota.py +98 -0
  32. data_factory/core/annotations/formats.py +626 -0
  33. data_factory/core/annotations/labelme.py +153 -0
  34. data_factory/core/annotations/model.py +334 -0
  35. data_factory/core/annotations/voc.py +231 -0
  36. data_factory/core/annotations/yolo.py +283 -0
  37. data_factory/core/bktree.py +89 -0
  38. data_factory/core/fs.py +245 -0
  39. data_factory/core/imaging.py +626 -0
  40. data_factory/core/ops/__init__.py +44 -0
  41. data_factory/core/ops/analyze.py +383 -0
  42. data_factory/core/ops/annotate_export.py +326 -0
  43. data_factory/core/ops/audit.py +274 -0
  44. data_factory/core/ops/augment.py +454 -0
  45. data_factory/core/ops/collect.py +241 -0
  46. data_factory/core/ops/combine.py +582 -0
  47. data_factory/core/ops/convert.py +212 -0
  48. data_factory/core/ops/dataset.py +246 -0
  49. data_factory/core/ops/dedup.py +235 -0
  50. data_factory/core/ops/export.py +168 -0
  51. data_factory/core/ops/image.py +407 -0
  52. data_factory/core/ops/labels.py +215 -0
  53. data_factory/core/ops/pairing.py +133 -0
  54. data_factory/core/ops/rename.py +154 -0
  55. data_factory/core/ops/report.py +323 -0
  56. data_factory/core/ops/visualize.py +164 -0
  57. data_factory/core/pairs.py +144 -0
  58. data_factory/core/parallel.py +191 -0
  59. data_factory/core/provenance.py +136 -0
  60. data_factory/core/registry.py +339 -0
  61. data_factory/core/stats.py +403 -0
  62. data_factory/core/types.py +148 -0
  63. data_factory/core/writer.py +194 -0
  64. data_factory/desktop.py +207 -0
  65. data_factory/hardware.py +243 -0
  66. data_factory/pipeline.py +192 -0
  67. data_factory/py.typed +0 -0
  68. data_factory/service/__init__.py +26 -0
  69. data_factory/service/app.py +483 -0
  70. data_factory/service/binding.py +174 -0
  71. data_factory/service/deps.py +133 -0
  72. data_factory/service/jobs.py +177 -0
  73. data_factory/service/limits.py +71 -0
  74. data_factory/service/routers/__init__.py +11 -0
  75. data_factory/service/routers/annotate.py +582 -0
  76. data_factory/service/routers/local.py +464 -0
  77. data_factory/service/routers/media.py +154 -0
  78. data_factory/service/routers/predictions.py +116 -0
  79. data_factory/service/routers/promote.py +182 -0
  80. data_factory/service/routers/tools.py +308 -0
  81. data_factory/service/static/annotate/api.js +246 -0
  82. data_factory/service/static/annotate/app.js +1696 -0
  83. data_factory/service/static/annotate/assist.js +106 -0
  84. data_factory/service/static/annotate/filmstrip.js +286 -0
  85. data_factory/service/static/annotate/hit.js +146 -0
  86. data_factory/service/static/annotate/labels.js +197 -0
  87. data_factory/service/static/annotate/render.js +286 -0
  88. data_factory/service/static/annotate/scene.js +277 -0
  89. data_factory/service/static/annotate/shapelist.js +72 -0
  90. data_factory/service/static/annotate/tools/box.js +96 -0
  91. data_factory/service/static/annotate/tools/line.js +96 -0
  92. data_factory/service/static/annotate/tools/obb.js +88 -0
  93. data_factory/service/static/annotate/tools/point.js +49 -0
  94. data_factory/service/static/annotate/tools/polygon.js +110 -0
  95. data_factory/service/static/annotate/tools/select.js +227 -0
  96. data_factory/service/static/annotate/undo.js +176 -0
  97. data_factory/service/static/annotate/viewport.js +156 -0
  98. data_factory/service/static/annotate.html +390 -0
  99. data_factory/service/static/index.html +632 -0
  100. data_factory/service/workspace.py +244 -0
  101. easyannotate/__init__.py +64 -0
  102. easyannotate/__main__.py +5 -0
  103. easyannotate/project.py +181 -0
  104. easyannotate/py.typed +0 -0
  105. easyannotate-0.4.0.dist-info/METADATA +767 -0
  106. easyannotate-0.4.0.dist-info/RECORD +109 -0
  107. easyannotate-0.4.0.dist-info/WHEEL +4 -0
  108. easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
  109. easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,488 @@
1
+ """Shapes: storage, the save diff, and the bridge back to the neutral model.
2
+
3
+ Two things here are load-bearing.
4
+
5
+ **Saves are a diff, not a replace.** The obvious implementation -- delete every
6
+ shape for the image and insert what arrived -- loses per-shape history on every
7
+ autosave. That history is what Phase 2 is built on: whether a shape came from a
8
+ model, and whether a person has since touched it, is the difference between a
9
+ prediction and a label. So incoming shapes carry their id, and a save resolves
10
+ to updates, inserts and deletes against what is already there.
11
+
12
+ **Saves are versioned.** Every image carries a version; a save states the
13
+ version it was based on. Two tabs open on the same image no longer means the
14
+ slower one silently wins -- the second gets a :class:`Conflict` and the server's
15
+ current state to reconcile against.
16
+
17
+ Everything leaves this module as :class:`~data_factory.core.annotations.model.Shape`,
18
+ the same type the five format writers already take. Export therefore needs no
19
+ new serialiser: it asks for annotations and hands them to the existing writers.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import json
25
+ import time
26
+ from dataclasses import dataclass, field
27
+ from typing import Any
28
+
29
+ from ..core.annotations.model import Annotation, Shape
30
+ from . import labels as labels_mod
31
+ from .images import Image
32
+ from .projects import Project, touch
33
+ from .store import Conflict, Store, StoreError, new_id
34
+
35
+ #: human -- somebody drew it. model -- a detector proposed it.
36
+ ORIGINS = ("human", "model")
37
+
38
+
39
+ @dataclass(slots=True)
40
+ class StoredShape:
41
+ """A shape as it exists in the database: geometry plus provenance."""
42
+
43
+ id: str
44
+ shape: Shape
45
+ label_idx: int
46
+ origin: str = "human"
47
+ created_at: float = 0.0
48
+ edited_at: float | None = None
49
+
50
+ @property
51
+ def is_model(self) -> bool:
52
+ return self.origin == "model"
53
+
54
+ @property
55
+ def is_touched(self) -> bool:
56
+ """A model shape a person has since corrected. Renders solid."""
57
+ return self.edited_at is not None
58
+
59
+ def as_dict(self) -> dict[str, Any]:
60
+ payload = self.shape.as_dict()
61
+ payload.update(
62
+ {
63
+ "id": self.id,
64
+ "label_idx": self.label_idx,
65
+ "origin": self.origin,
66
+ "edited": self.is_touched,
67
+ }
68
+ )
69
+ return payload
70
+
71
+
72
+ def _geometry(row: Any) -> tuple[tuple[int, int, int, int], list[tuple[int, int]] | None]:
73
+ bbox = (int(row["xmin"]), int(row["ymin"]), int(row["xmax"]), int(row["ymax"]))
74
+ raw = row["polygon"]
75
+ if not raw:
76
+ return bbox, None
77
+ try:
78
+ points = json.loads(raw)
79
+ except (TypeError, ValueError):
80
+ # A corrupt polygon column should degrade to a box, not lose the
81
+ # object entirely. The box columns are always authoritative.
82
+ return bbox, None
83
+ return bbox, [(int(x), int(y)) for x, y in points]
84
+
85
+
86
+ def _stored(row: Any, names: dict[int, str]) -> StoredShape:
87
+ bbox, polygon = _geometry(row)
88
+ idx = int(row["label_idx"])
89
+ return StoredShape(
90
+ id=row["id"],
91
+ shape=Shape(
92
+ label=names.get(idx, f"class_{idx}"),
93
+ bbox=bbox,
94
+ polygon=polygon,
95
+ pose=row["pose"],
96
+ truncated=int(row["truncated"]),
97
+ difficult=int(row["difficult"]),
98
+ occluded=int(row["occluded"]),
99
+ confidence=row["confidence"],
100
+ kind=str(row["kind"] or "") if "kind" in row.keys() else "",
101
+ ),
102
+ label_idx=idx,
103
+ origin=row["origin"],
104
+ created_at=float(row["created_at"]),
105
+ edited_at=row["edited_at"],
106
+ )
107
+
108
+
109
+ def _name_map(store: Store, project: Project) -> dict[int, str]:
110
+ return {
111
+ int(r["idx"]): str(r["name"])
112
+ for r in store.query("SELECT idx, name FROM label WHERE project_id = ?", (project.id,))
113
+ }
114
+
115
+
116
+ def listing(
117
+ store: Store, project: Project, image: Image, *, names: dict[int, str] | None = None
118
+ ) -> list[StoredShape]:
119
+ names = names if names is not None else _name_map(store, project)
120
+ rows = store.query(
121
+ "SELECT * FROM shape WHERE image_id = ? ORDER BY created_at, id", (image.id,)
122
+ )
123
+ return [_stored(r, names) for r in rows]
124
+
125
+
126
+ def annotation_for(store: Store, project: Project, image: Image) -> Annotation:
127
+ """Everything known about one image, as the format writers want it."""
128
+ return Annotation(
129
+ filename=image.name,
130
+ width=image.width,
131
+ height=image.height,
132
+ # Export names, which are unique: the writers map shapes to class
133
+ # indices by name, and a retired class sharing a live one's name
134
+ # would otherwise be merged into it in the output.
135
+ shapes=[
136
+ s.shape
137
+ for s in listing(store, project, image, names=labels_mod.export_names(store, project))
138
+ ],
139
+ )
140
+
141
+
142
+ @dataclass(slots=True)
143
+ class SaveResult:
144
+ version: int
145
+ inserted: int = 0
146
+ updated: int = 0
147
+ deleted: int = 0
148
+ labels_created: list[str] = field(default_factory=list)
149
+ #: The stored id of each submitted shape, **in submission order**.
150
+ #:
151
+ #: This is what lets a client turn its local drafts into real shapes after
152
+ #: a save. Without it the next save would send those shapes with no id
153
+ #: again and they would insert a second time -- every autosave silently
154
+ #: doubling the boxes on the image. Order is the mapping, because it is the
155
+ #: only thing both sides already agree on.
156
+ ids: list[str] = field(default_factory=list)
157
+
158
+
159
+ def _same_geometry(stored: StoredShape, shape: Shape) -> bool:
160
+ return stored.shape.bbox == shape.bbox and stored.shape.polygon == shape.polygon
161
+
162
+
163
+ def save(
164
+ store: Store,
165
+ project: Project,
166
+ image: Image,
167
+ incoming: list[tuple[str | None, Shape]],
168
+ *,
169
+ base_version: int | None = None,
170
+ origin: str = "human",
171
+ advance_state: bool = True,
172
+ ) -> SaveResult:
173
+ """Apply the editor's shape list to one image.
174
+
175
+ ``incoming`` is ``(id or None, shape)`` pairs: an id means "this is the
176
+ shape you already have", ``None`` means "this one is new". Anything in the
177
+ database and absent from the list is deleted.
178
+
179
+ Labels are created on demand from the shape's name -- this is the whole
180
+ "draw, type a name, Enter" promise, and the reason there is no schema step.
181
+ """
182
+ if origin not in ORIGINS:
183
+ raise StoreError(f"origin must be one of {', '.join(ORIGINS)}")
184
+
185
+ if base_version is not None and int(base_version) != image.version:
186
+ raise _moved_on(base_version, image.version)
187
+
188
+ # Labels first, outside the shape transaction. `ensure` takes the write
189
+ # lock itself, and creating a label is a legitimate side effect of a save
190
+ # even when the geometry turns out to be a no-op.
191
+ #
192
+ # Only names that *changed* are resolved. The editor sends names, not
193
+ # indices, and re-resolving an untouched shape's name would move every
194
+ # shape of a retired class into the live class sharing its name -- or
195
+ # recreate the class -- on the next autosave of its image.
196
+ created: list[str] = []
197
+ idx_for: dict[str, int] = {}
198
+ before_edit = {s.id: s for s in listing(store, project, image)}
199
+ for shape_id, shape in incoming:
200
+ if shape.label in idx_for or _keeps_class(before_edit.get(shape_id), shape):
201
+ continue
202
+ before = labels_mod.by_name(store, project, shape.label)
203
+ label = labels_mod.ensure(store, project, shape.label)
204
+ idx_for[shape.label] = label.idx
205
+ # `ensure` normalises whitespace, so match on the resolved name.
206
+ idx_for[label.name] = label.idx
207
+ if before is None:
208
+ created.append(label.name)
209
+
210
+ seen: set[str] = set()
211
+ now = time.time()
212
+ model_inserts = 0
213
+
214
+ with store.write() as conn:
215
+ # Check and read again *inside* the transaction. The check above is
216
+ # only a fast path: between it and here another save -- a second tab,
217
+ # or an autosave racing a navigate-flush -- can commit, and a diff
218
+ # computed against what it replaced would resurrect deleted shapes or
219
+ # drop new ones without anyone being told.
220
+ row = conn.execute(
221
+ "SELECT version, state FROM image WHERE id = ?", (image.id,)
222
+ ).fetchone()
223
+ if row is None:
224
+ raise StoreError("that image no longer exists")
225
+ image.version, image.state = int(row["version"]), row["state"]
226
+ if base_version is not None and int(base_version) != image.version:
227
+ raise _moved_on(base_version, image.version)
228
+ existing = {s.id: s for s in listing(store, project, image)}
229
+ result = SaveResult(version=image.version, labels_created=created)
230
+
231
+ for position, (shape_id, raw) in enumerate(incoming):
232
+ # Server-side clamping, always. A browser can send a box dragged
233
+ # off the edge of the image, and every format writer downstream
234
+ # assumes coordinates are inside the picture.
235
+ shape = raw.clamped(image.width, image.height) if image.width and image.height else raw
236
+ before = existing.get(shape_id) if shape_id else None
237
+ if _keeps_class(before, shape):
238
+ label_idx = before.label_idx
239
+ elif shape.label in idx_for or labels_mod.clean_name(shape.label) in idx_for:
240
+ label_idx = idx_for.get(shape.label, idx_for.get(labels_mod.clean_name(shape.label)))
241
+ else:
242
+ # Its class changed between the read above and this lock:
243
+ # another save got in. Ask the client to reload, never guess.
244
+ raise _moved_on(base_version, image.version)
245
+ polygon = (
246
+ json.dumps([[int(x), int(y)] for x, y in shape.polygon])
247
+ if shape.polygon
248
+ else None
249
+ )
250
+ xmin, ymin, xmax, ymax = shape.bbox
251
+
252
+ prior = existing.get(shape_id) if shape_id else None
253
+ if prior is not None:
254
+ seen.add(prior.id)
255
+ # A model shape a person has moved becomes their shape: it
256
+ # stops rendering dashed and starts counting as a correction.
257
+ edited = prior.edited_at
258
+ if prior.is_model and not _same_geometry(prior, shape):
259
+ edited = now
260
+ conn.execute(
261
+ "UPDATE shape SET label_idx = ?, xmin = ?, ymin = ?, xmax = ?,"
262
+ " ymax = ?, polygon = ?, pose = ?, truncated = ?, difficult = ?,"
263
+ " occluded = ?, confidence = ?, edited_at = ?, kind = ? WHERE id = ?",
264
+ (
265
+ label_idx, xmin, ymin, xmax, ymax, polygon, shape.pose,
266
+ shape.truncated, shape.difficult, shape.occluded,
267
+ shape.confidence, edited, shape.shape_kind, prior.id,
268
+ ),
269
+ )
270
+ result.updated += 1
271
+ result.ids.append(prior.id)
272
+ else:
273
+ fresh = new_id()
274
+ # A new shape that carries a confidence came from a model --
275
+ # a text prompt run on one image, say -- and stays a
276
+ # prediction: dashed, reviewable, counted by agreement. Saving
277
+ # it through the editor used to turn it into hand-drawn work.
278
+ row_origin = origin
279
+ if origin == "human" and shape.confidence is not None:
280
+ row_origin = "model"
281
+ # Only a shape the server has never seen is a new
282
+ # prediction. One coming back under an old id is an undone
283
+ # delete: it was counted when it first arrived.
284
+ if shape_id is None:
285
+ model_inserts += 1
286
+ conn.execute(
287
+ "INSERT INTO shape (id, image_id, project_id, label_idx, xmin,"
288
+ " ymin, xmax, ymax, polygon, origin, confidence, pose, truncated,"
289
+ " difficult, occluded, created_at, kind)"
290
+ " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
291
+ (
292
+ fresh, image.id, project.id, label_idx, xmin, ymin, xmax,
293
+ ymax, polygon, row_origin, shape.confidence, shape.pose,
294
+ shape.truncated, shape.difficult, shape.occluded,
295
+ # Distinct per shape, in submission order. Listing
296
+ # sorts by created_at, and shapes saved in one burst
297
+ # would otherwise come back in random id order --
298
+ # reshuffling what is on top of what after a reload.
299
+ now + position * 1e-6,
300
+ shape.shape_kind,
301
+ ),
302
+ )
303
+ result.inserted += 1
304
+ result.ids.append(fresh)
305
+
306
+ for gone in set(existing) - seen:
307
+ conn.execute("DELETE FROM shape WHERE id = ?", (gone,))
308
+ result.deleted += 1
309
+
310
+ result.version = image.version + 1
311
+ state = image.state
312
+ if advance_state and state in ("todo", "predicted"):
313
+ state = "in_progress"
314
+ if model_inserts:
315
+ # Predictions the editor added count in agreement's denominator,
316
+ # so deleting one later still registers as a correction.
317
+ conn.execute(
318
+ "UPDATE image SET predicted_count = predicted_count + ? WHERE id = ?",
319
+ (model_inserts, image.id),
320
+ )
321
+ updated = conn.execute(
322
+ "UPDATE image SET version = ?, state = ?, updated_at = ?"
323
+ " WHERE id = ? AND version = ?",
324
+ (result.version, state, now, image.id, image.version),
325
+ )
326
+ if updated.rowcount != 1: # unreachable under the write lock; kept as the backstop
327
+ raise _moved_on(image.version, None)
328
+ touch(conn, project.id)
329
+
330
+ image.version = result.version
331
+ image.state = state
332
+ refresh_uncertainty(store, project, image)
333
+ return result
334
+
335
+
336
+ #: Two predictions of different classes overlapping this much are the model
337
+ #: arguing with itself about what the object is.
338
+ CONFLICT_IOU = 0.7
339
+
340
+
341
+ def uncertainty_of(stored: list[StoredShape]) -> float:
342
+ """How much review an image's predictions need. Higher: look sooner.
343
+
344
+ Only untouched model shapes count -- what a person drew or corrected is
345
+ settled. Each scores its ambiguity, 1 at confidence 0.5 falling to 0 at
346
+ 0 or 1, so a wall of 0.95s is quick work and a handful of 0.5s is not.
347
+ Every pair of overlapping predictions with different classes adds 1:
348
+ class confusion is the commonest systematic error and the most
349
+ expensive one to train on.
350
+ """
351
+ return score_rows(
352
+ (s.label_idx, *s.shape.bbox, s.shape.confidence)
353
+ for s in stored
354
+ if s.is_model and not s.is_touched
355
+ )
356
+
357
+
358
+ def score_rows(rows) -> float:
359
+ """The score from bare (label_idx, x1, y1, x2, y2, confidence) rows --
360
+ shared with the migration that backfills it."""
361
+ live = [tuple(r) for r in rows]
362
+ score = 0.0
363
+ for row in live:
364
+ if row[5] is not None:
365
+ score += 1.0 - abs(2.0 * float(row[5]) - 1.0)
366
+ for i, a in enumerate(live):
367
+ for b in live[i + 1:]:
368
+ if a[0] != b[0] and _iou(a[1:5], b[1:5]) > CONFLICT_IOU:
369
+ score += 1.0
370
+ return round(score, 4)
371
+
372
+
373
+ def _iou(a: tuple[int, int, int, int], b: tuple[int, int, int, int]) -> float:
374
+ ix = max(0, min(a[2], b[2]) - max(a[0], b[0]))
375
+ iy = max(0, min(a[3], b[3]) - max(a[1], b[1]))
376
+ inter = ix * iy
377
+ union = (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - inter
378
+ return inter / union if union > 0 else 0.0
379
+
380
+
381
+ def refresh_uncertainty(store: Store, project: Project, image: Image) -> float:
382
+ score = uncertainty_of(listing(store, project, image))
383
+ with store.write() as conn:
384
+ conn.execute("UPDATE image SET uncertainty = ? WHERE id = ?", (score, image.id))
385
+ image.uncertainty = score
386
+ return score
387
+
388
+
389
+ def _keeps_class(before: StoredShape | None, shape: Shape) -> bool:
390
+ """An existing shape submitted under the name it already had."""
391
+ return before is not None and before.shape.label == shape.label
392
+
393
+
394
+ def _moved_on(had: int | None, now: int | None) -> Conflict:
395
+ current = "" if now is None else f", it is now {now}"
396
+ return Conflict(
397
+ f"this image moved on (you had version {had}{current}). "
398
+ "Reload and reapply your change."
399
+ )
400
+
401
+
402
+ def replace_all(
403
+ store: Store,
404
+ project: Project,
405
+ image: Image,
406
+ shapes: list[Shape],
407
+ *,
408
+ origin: str = "human",
409
+ base_version: int | None = None,
410
+ ) -> SaveResult:
411
+ """Save a list of plain shapes, treating all of them as new.
412
+
413
+ The import path -- predictions, or a folder of existing sidecar files --
414
+ where nothing has an id yet.
415
+ """
416
+ return save(
417
+ store,
418
+ project,
419
+ image,
420
+ [(None, s) for s in shapes],
421
+ base_version=base_version,
422
+ origin=origin,
423
+ advance_state=False,
424
+ )
425
+
426
+
427
+ def label_histogram(store: Store, project: Project) -> dict[str, int]:
428
+ """Objects per class, by name. Feeds the class-balance panel."""
429
+ names = _name_map(store, project)
430
+ out: dict[str, int] = {}
431
+ for row in store.query(
432
+ "SELECT label_idx, COUNT(*) AS n FROM shape WHERE project_id = ? GROUP BY label_idx",
433
+ (project.id,),
434
+ ):
435
+ idx = int(row["label_idx"])
436
+ out[names.get(idx, f"class_{idx}")] = int(row["n"])
437
+ return out
438
+
439
+
440
+ def agreement(store: Store, project: Project) -> dict[str, Any]:
441
+ """How often the model was right, as a headline number.
442
+
443
+ Counts model shapes that survived review untouched against those a person
444
+ moved or deleted. This is the figure that tells you when to retrain and
445
+ when to stop annotating by hand.
446
+
447
+ **Scoped to reviewed images.** A prediction nobody has looked at is not
448
+ evidence of agreement -- it is evidence of nothing. Counting the whole
449
+ project would mean importing a fresh run made the model look better,
450
+ because every new unexamined guess would score as an agreement, and the
451
+ number would climb fastest exactly when the least work had been done.
452
+ """
453
+ # The denominator is what the model *predicted* on reviewed images, not
454
+ # what survived -- see the predicted_count column. Deleting a false
455
+ # positive is the commonest correction there is, and counting only
456
+ # survivors would score a model that hallucinates boxes at 100%.
457
+ predicted = store.one(
458
+ "SELECT COALESCE(SUM(predicted_count), 0) AS n FROM image"
459
+ " WHERE project_id = ? AND state = 'reviewed'",
460
+ (project.id,),
461
+ )
462
+ total = int(predicted["n"] or 0)
463
+
464
+ row = store.one(
465
+ "SELECT COUNT(*) AS surviving,"
466
+ " SUM(CASE WHEN s.edited_at IS NOT NULL THEN 1 ELSE 0 END) AS edited"
467
+ " FROM shape s JOIN image i ON i.id = s.image_id"
468
+ " WHERE s.project_id = ? AND s.origin = 'model' AND i.state = 'reviewed'",
469
+ (project.id,),
470
+ )
471
+ surviving = int(row["surviving"] or 0)
472
+ edited = int(row["edited"] or 0)
473
+ deleted = max(0, total - surviving)
474
+ corrected = edited + deleted
475
+
476
+ pending = store.one(
477
+ "SELECT COALESCE(SUM(predicted_count), 0) AS n FROM image"
478
+ " WHERE project_id = ? AND state != 'reviewed'",
479
+ (project.id,),
480
+ )
481
+ return {
482
+ "model_shapes": total,
483
+ "corrected": corrected,
484
+ "adjusted": edited,
485
+ "deleted": deleted,
486
+ "unreviewed": int(pending["n"] or 0),
487
+ "agreement": round(max(0.0, 1.0 - (corrected / total)), 4) if total else None,
488
+ }