easyannotate 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_factory/__init__.py +82 -0
- data_factory/__main__.py +6 -0
- data_factory/annotate/__init__.py +15 -0
- data_factory/annotate/attach.py +398 -0
- data_factory/annotate/demo.py +161 -0
- data_factory/annotate/images.py +443 -0
- data_factory/annotate/labels.py +399 -0
- data_factory/annotate/loop.py +359 -0
- data_factory/annotate/projects.py +207 -0
- data_factory/annotate/review.py +471 -0
- data_factory/annotate/schema.sql +171 -0
- data_factory/annotate/shapes.py +488 -0
- data_factory/annotate/store.py +292 -0
- data_factory/annotate/tools/__init__.py +39 -0
- data_factory/annotate/tools/fake.py +170 -0
- data_factory/annotate/tools/install.py +254 -0
- data_factory/annotate/tools/local.py +389 -0
- data_factory/annotate/tools/postprocess.py +175 -0
- data_factory/annotate/tools/protocol.py +238 -0
- data_factory/annotate/tools/registry.py +196 -0
- data_factory/annotate/tools/results.py +83 -0
- data_factory/annotate/tools/serve.py +106 -0
- data_factory/cli/__init__.py +21 -0
- data_factory/cli/main.py +837 -0
- data_factory/cli/menu.py +173 -0
- data_factory/cli/render.py +158 -0
- data_factory/core/__init__.py +11 -0
- data_factory/core/annotations/__init__.py +5 -0
- data_factory/core/annotations/coco.py +394 -0
- data_factory/core/annotations/csvfmt.py +174 -0
- data_factory/core/annotations/dota.py +98 -0
- data_factory/core/annotations/formats.py +626 -0
- data_factory/core/annotations/labelme.py +153 -0
- data_factory/core/annotations/model.py +334 -0
- data_factory/core/annotations/voc.py +231 -0
- data_factory/core/annotations/yolo.py +283 -0
- data_factory/core/bktree.py +89 -0
- data_factory/core/fs.py +245 -0
- data_factory/core/imaging.py +626 -0
- data_factory/core/ops/__init__.py +44 -0
- data_factory/core/ops/analyze.py +383 -0
- data_factory/core/ops/annotate_export.py +326 -0
- data_factory/core/ops/audit.py +274 -0
- data_factory/core/ops/augment.py +454 -0
- data_factory/core/ops/collect.py +241 -0
- data_factory/core/ops/combine.py +582 -0
- data_factory/core/ops/convert.py +212 -0
- data_factory/core/ops/dataset.py +246 -0
- data_factory/core/ops/dedup.py +235 -0
- data_factory/core/ops/export.py +168 -0
- data_factory/core/ops/image.py +407 -0
- data_factory/core/ops/labels.py +215 -0
- data_factory/core/ops/pairing.py +133 -0
- data_factory/core/ops/rename.py +154 -0
- data_factory/core/ops/report.py +323 -0
- data_factory/core/ops/visualize.py +164 -0
- data_factory/core/pairs.py +144 -0
- data_factory/core/parallel.py +191 -0
- data_factory/core/provenance.py +136 -0
- data_factory/core/registry.py +339 -0
- data_factory/core/stats.py +403 -0
- data_factory/core/types.py +148 -0
- data_factory/core/writer.py +194 -0
- data_factory/desktop.py +207 -0
- data_factory/hardware.py +243 -0
- data_factory/pipeline.py +192 -0
- data_factory/py.typed +0 -0
- data_factory/service/__init__.py +26 -0
- data_factory/service/app.py +483 -0
- data_factory/service/binding.py +174 -0
- data_factory/service/deps.py +133 -0
- data_factory/service/jobs.py +177 -0
- data_factory/service/limits.py +71 -0
- data_factory/service/routers/__init__.py +11 -0
- data_factory/service/routers/annotate.py +582 -0
- data_factory/service/routers/local.py +464 -0
- data_factory/service/routers/media.py +154 -0
- data_factory/service/routers/predictions.py +116 -0
- data_factory/service/routers/promote.py +182 -0
- data_factory/service/routers/tools.py +308 -0
- data_factory/service/static/annotate/api.js +246 -0
- data_factory/service/static/annotate/app.js +1696 -0
- data_factory/service/static/annotate/assist.js +106 -0
- data_factory/service/static/annotate/filmstrip.js +286 -0
- data_factory/service/static/annotate/hit.js +146 -0
- data_factory/service/static/annotate/labels.js +197 -0
- data_factory/service/static/annotate/render.js +286 -0
- data_factory/service/static/annotate/scene.js +277 -0
- data_factory/service/static/annotate/shapelist.js +72 -0
- data_factory/service/static/annotate/tools/box.js +96 -0
- data_factory/service/static/annotate/tools/line.js +96 -0
- data_factory/service/static/annotate/tools/obb.js +88 -0
- data_factory/service/static/annotate/tools/point.js +49 -0
- data_factory/service/static/annotate/tools/polygon.js +110 -0
- data_factory/service/static/annotate/tools/select.js +227 -0
- data_factory/service/static/annotate/undo.js +176 -0
- data_factory/service/static/annotate/viewport.js +156 -0
- data_factory/service/static/annotate.html +390 -0
- data_factory/service/static/index.html +632 -0
- data_factory/service/workspace.py +244 -0
- easyannotate/__init__.py +64 -0
- easyannotate/__main__.py +5 -0
- easyannotate/project.py +181 -0
- easyannotate/py.typed +0 -0
- easyannotate-0.4.0.dist-info/METADATA +767 -0
- easyannotate-0.4.0.dist-info/RECORD +109 -0
- easyannotate-0.4.0.dist-info/WHEEL +4 -0
- easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
- easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,471 @@
|
|
|
1
|
+
"""Model-in-the-loop review: getting predictions back in, and judging them.
|
|
2
|
+
|
|
3
|
+
The loop this closes: annotate a few hundred images by hand, train, run
|
|
4
|
+
inference on the rest, **load those predictions back in**, fix the wrong ones,
|
|
5
|
+
export, retrain. Each turn costs less than the one before, and the only part
|
|
6
|
+
that was missing is this one.
|
|
7
|
+
|
|
8
|
+
Three rules, each of which exists because the obvious alternative fails
|
|
9
|
+
quietly:
|
|
10
|
+
|
|
11
|
+
**Matching is reported, never assumed.** Predictions are matched to images by
|
|
12
|
+
filename stem, and the import returns exactly how many matched and what did
|
|
13
|
+
not. Silently importing zero shapes because ``img_001.txt`` met ``IMG_001.jpg``
|
|
14
|
+
is the single most common way this feature disappoints people, and it looks
|
|
15
|
+
identical to a model that predicted nothing.
|
|
16
|
+
|
|
17
|
+
**An import replaces the previous run's predictions, not your work.** Model
|
|
18
|
+
shapes from an earlier run are cleared; hand-drawn shapes, and model shapes a
|
|
19
|
+
person has already corrected, are kept. Otherwise a second run doubles every
|
|
20
|
+
box on the image.
|
|
21
|
+
|
|
22
|
+
**Reviewed images are left alone by default.** Dropping fresh predictions onto
|
|
23
|
+
an image somebody already signed off would quietly un-finish it.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import json
|
|
29
|
+
import time
|
|
30
|
+
from collections.abc import Callable
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from typing import Any
|
|
33
|
+
|
|
34
|
+
from ..core.annotations import coco as coco_fmt
|
|
35
|
+
from ..core.annotations import formats, yolo
|
|
36
|
+
from ..core.annotations.model import Annotation, AnnotationError
|
|
37
|
+
from . import images as images_mod
|
|
38
|
+
from . import labels as labels_mod
|
|
39
|
+
from . import projects as projects_mod
|
|
40
|
+
from . import shapes as shapes_mod
|
|
41
|
+
from .store import Store, StoreError, new_id
|
|
42
|
+
|
|
43
|
+
#: States an import will not touch unless explicitly told to.
|
|
44
|
+
PROTECTED_STATES = ("reviewed",)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class ReviewError(StoreError):
|
|
48
|
+
"""Raised when predictions cannot be read or matched."""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def read_predictions(
|
|
52
|
+
source: Path,
|
|
53
|
+
*,
|
|
54
|
+
project_classes: list[str] | None = None,
|
|
55
|
+
sizes: dict[str, tuple[int, int]] | None = None,
|
|
56
|
+
locate: Callable[[str], Any] | None = None,
|
|
57
|
+
) -> tuple[list[Annotation], list[str]]:
|
|
58
|
+
"""Read a folder or file of model output into annotations.
|
|
59
|
+
|
|
60
|
+
Handles the shapes real tools actually emit:
|
|
61
|
+
|
|
62
|
+
* a folder of YOLO ``.txt`` with a trailing confidence column
|
|
63
|
+
* a folder of VOC ``.xml`` or LabelMe ``.json``
|
|
64
|
+
* a COCO dataset json
|
|
65
|
+
* a COCO **detection results** json -- a bare list, which is what
|
|
66
|
+
torchvision, mmdetection and Ultralytics evaluation scripts write
|
|
67
|
+
|
|
68
|
+
``sizes`` maps a filename stem to the image's ``(width, height)`` and comes
|
|
69
|
+
from the project. It is needed because **a prediction folder has no images
|
|
70
|
+
in it** -- an inference run writes labels, not pictures -- and YOLO
|
|
71
|
+
coordinates are normalised, so without the size there is nothing to
|
|
72
|
+
multiply by. ``locate``, when given, resolves a label file's relative
|
|
73
|
+
path to the project image (anything with ``width``/``height``) and is
|
|
74
|
+
preferred over ``sizes``, which can only key on the stem. The ordinary dataset reader pairs labels with images and
|
|
75
|
+
therefore cannot read this at all; that is why it is not simply reused.
|
|
76
|
+
"""
|
|
77
|
+
notes: list[str] = []
|
|
78
|
+
source = Path(source)
|
|
79
|
+
sizes = sizes or {}
|
|
80
|
+
|
|
81
|
+
if source.is_file() and source.suffix.lower() == ".json":
|
|
82
|
+
return _read_json(source, project_classes, notes)
|
|
83
|
+
|
|
84
|
+
if not source.is_dir():
|
|
85
|
+
raise ReviewError(f"{source} is not a folder or a .json file")
|
|
86
|
+
|
|
87
|
+
# A results json sitting alone in a folder is the common case when someone
|
|
88
|
+
# zips up "the output", so look for it before anything else.
|
|
89
|
+
for candidate in sorted(source.rglob("*.json")):
|
|
90
|
+
if not candidate.is_file():
|
|
91
|
+
continue
|
|
92
|
+
try:
|
|
93
|
+
document = json.loads(candidate.read_text(encoding="utf-8"))
|
|
94
|
+
except (OSError, ValueError):
|
|
95
|
+
continue
|
|
96
|
+
if coco_fmt.is_detection_results(document):
|
|
97
|
+
notes.append(f"read COCO detection results from {candidate.name}")
|
|
98
|
+
return (
|
|
99
|
+
coco_fmt.results_to_annotations(
|
|
100
|
+
document, names=_names_for(project_classes)
|
|
101
|
+
),
|
|
102
|
+
notes,
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
label_files = [p for p in sorted(source.rglob("*.txt")) if p.is_file()]
|
|
106
|
+
if label_files:
|
|
107
|
+
return _read_yolo_labels(source, label_files, project_classes, sizes, notes, locate)
|
|
108
|
+
|
|
109
|
+
try:
|
|
110
|
+
dataset = formats.read_dataset(source, "auto", recursive=True)
|
|
111
|
+
except AnnotationError as exc:
|
|
112
|
+
raise ReviewError(f"could not read predictions: {exc}") from exc
|
|
113
|
+
|
|
114
|
+
notes.extend(dataset.warnings[:10])
|
|
115
|
+
notes.append(f"read {dataset.source_format} predictions")
|
|
116
|
+
return [item.annotation for item in dataset.items], notes
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _read_yolo_labels(
|
|
120
|
+
source: Path,
|
|
121
|
+
label_files: list[Path],
|
|
122
|
+
project_classes: list[str] | None,
|
|
123
|
+
sizes: dict[str, tuple[int, int]],
|
|
124
|
+
notes: list[str],
|
|
125
|
+
locate: Callable[[str], Any] | None = None,
|
|
126
|
+
) -> tuple[list[Annotation], list[str]]:
|
|
127
|
+
"""Read a folder of bare YOLO ``.txt`` predictions.
|
|
128
|
+
|
|
129
|
+
The class list comes from a ``data.yaml`` if the run shipped one, and
|
|
130
|
+
otherwise from the project -- which is the right guess, because a model
|
|
131
|
+
trained from this project was trained on this project's class order. When
|
|
132
|
+
neither is available the names come out as ``class_0``, visibly wrong
|
|
133
|
+
rather than quietly attributed to the wrong class.
|
|
134
|
+
"""
|
|
135
|
+
yaml_path = yolo.find_data_yaml(source)
|
|
136
|
+
if yaml_path is not None:
|
|
137
|
+
try:
|
|
138
|
+
classes, _ = yolo.read_data_yaml(yaml_path)
|
|
139
|
+
notes.append(f"class order read from {yaml_path.name}")
|
|
140
|
+
except AnnotationError:
|
|
141
|
+
classes = yolo.ClassList(list(project_classes or []))
|
|
142
|
+
notes.append("data.yaml was unreadable; used the project's class order")
|
|
143
|
+
elif project_classes:
|
|
144
|
+
classes = yolo.ClassList(list(project_classes))
|
|
145
|
+
notes.append("no data.yaml in the predictions; used the project's class order")
|
|
146
|
+
else:
|
|
147
|
+
classes = yolo.ClassList([])
|
|
148
|
+
notes.append(
|
|
149
|
+
"no data.yaml and no classes in the project, so classes are named "
|
|
150
|
+
"class_0, class_1 ... -- rename them after importing"
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
annotations: list[Annotation] = []
|
|
154
|
+
unmeasured = 0
|
|
155
|
+
for path in label_files:
|
|
156
|
+
stem = path.stem
|
|
157
|
+
# The relative path, not the stem, names the prediction: it is what
|
|
158
|
+
# tells val/001.txt from train/001.txt when matching it to an image.
|
|
159
|
+
name = path.relative_to(source).as_posix()
|
|
160
|
+
if locate is not None:
|
|
161
|
+
image = locate(name)
|
|
162
|
+
known = image is not None
|
|
163
|
+
width, height = (image.width, image.height) if known else (0, 0)
|
|
164
|
+
else:
|
|
165
|
+
known = stem in sizes
|
|
166
|
+
width, height = sizes.get(stem, (0, 0))
|
|
167
|
+
|
|
168
|
+
if not known:
|
|
169
|
+
# This file matches no image in the project. It still has to appear
|
|
170
|
+
# in the result, empty, so the caller can *report* it as unmatched
|
|
171
|
+
# -- dropping it here would turn "your filenames do not line up"
|
|
172
|
+
# into "no predictions were found", which sends people looking in
|
|
173
|
+
# entirely the wrong place.
|
|
174
|
+
annotations.append(
|
|
175
|
+
Annotation(filename=name, width=0, height=0, shapes=[])
|
|
176
|
+
)
|
|
177
|
+
continue
|
|
178
|
+
|
|
179
|
+
if not (width and height):
|
|
180
|
+
# The image is ours but we never managed to measure it. Normalised
|
|
181
|
+
# coordinates cannot be turned into pixels without a size, and
|
|
182
|
+
# guessing one would put every box in the top-left corner.
|
|
183
|
+
unmeasured += 1
|
|
184
|
+
continue
|
|
185
|
+
|
|
186
|
+
try:
|
|
187
|
+
annotations.append(
|
|
188
|
+
yolo.read(path, classes, width, height, filename=name)
|
|
189
|
+
)
|
|
190
|
+
except AnnotationError as exc:
|
|
191
|
+
notes.append(f"{path.name}: {exc}")
|
|
192
|
+
|
|
193
|
+
if unmeasured:
|
|
194
|
+
notes.append(
|
|
195
|
+
f"{unmeasured} image(s) have no recorded size, so their predictions "
|
|
196
|
+
f"could not be scaled and were skipped"
|
|
197
|
+
)
|
|
198
|
+
notes.append("read YOLO predictions")
|
|
199
|
+
return annotations, notes
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _read_json(
|
|
203
|
+
path: Path, project_classes: list[str] | None, notes: list[str]
|
|
204
|
+
) -> tuple[list[Annotation], list[str]]:
|
|
205
|
+
try:
|
|
206
|
+
document = json.loads(path.read_text(encoding="utf-8"))
|
|
207
|
+
except (OSError, ValueError) as exc:
|
|
208
|
+
raise ReviewError(f"{path.name}: not readable JSON ({exc})") from exc
|
|
209
|
+
|
|
210
|
+
if coco_fmt.is_detection_results(document):
|
|
211
|
+
notes.append("read COCO detection results (a bare list of predictions)")
|
|
212
|
+
return (
|
|
213
|
+
coco_fmt.results_to_annotations(document, names=_names_for(project_classes)),
|
|
214
|
+
notes,
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
try:
|
|
218
|
+
annotations = coco_fmt.to_annotations(document, unsupported_out=notes)
|
|
219
|
+
except AnnotationError as exc:
|
|
220
|
+
raise ReviewError(str(exc)) from exc
|
|
221
|
+
notes.append("read a COCO dataset document")
|
|
222
|
+
return annotations, notes
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _names_for(project_classes: list[str] | None) -> dict[Any, str]:
|
|
226
|
+
"""Map COCO category ids onto the project's class names.
|
|
227
|
+
|
|
228
|
+
A detection-results file carries category *ids* and no names. Ultralytics
|
|
229
|
+
and torchvision both number them from 0 in the order of the training class
|
|
230
|
+
list, which for a model trained from this project is the project's own
|
|
231
|
+
order -- so that is what we assume, and the matching report is what catches
|
|
232
|
+
it when the assumption is wrong.
|
|
233
|
+
"""
|
|
234
|
+
if not project_classes:
|
|
235
|
+
return {}
|
|
236
|
+
mapping: dict[Any, str] = {}
|
|
237
|
+
for index, name in enumerate(project_classes):
|
|
238
|
+
mapping[index] = name
|
|
239
|
+
# COCO's own convention numbers categories from 1.
|
|
240
|
+
mapping.setdefault(index + 1, name)
|
|
241
|
+
return mapping
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def import_predictions(
|
|
245
|
+
store: Store,
|
|
246
|
+
project: projects_mod.Project,
|
|
247
|
+
source: Path,
|
|
248
|
+
*,
|
|
249
|
+
name: str = "",
|
|
250
|
+
min_confidence: float = 0.0,
|
|
251
|
+
overwrite_reviewed: bool = False,
|
|
252
|
+
) -> dict[str, Any]:
|
|
253
|
+
"""Load a model's output onto a project's images.
|
|
254
|
+
|
|
255
|
+
Returns a matching report. Read it: the interesting number is usually the
|
|
256
|
+
one that did *not* match.
|
|
257
|
+
"""
|
|
258
|
+
class_list = labels_mod.class_list(store, project)
|
|
259
|
+
every_image = images_mod.listing_all(store, project)
|
|
260
|
+
# A prediction folder holds labels, not images, so the sizes YOLO needs to
|
|
261
|
+
# denormalise its coordinates have to come from the project -- found by
|
|
262
|
+
# the same matcher that later attaches the shapes, so the two agree.
|
|
263
|
+
matcher = images_mod.Matcher(every_image)
|
|
264
|
+
|
|
265
|
+
annotations, notes = read_predictions(
|
|
266
|
+
source, project_classes=class_list, locate=matcher.find
|
|
267
|
+
)
|
|
268
|
+
if not annotations:
|
|
269
|
+
raise ReviewError(
|
|
270
|
+
"no predictions were found in that file or folder. "
|
|
271
|
+
+ ("; ".join(notes) if notes else "")
|
|
272
|
+
)
|
|
273
|
+
matched = 0
|
|
274
|
+
skipped_reviewed = 0
|
|
275
|
+
unmatched_files: list[str] = []
|
|
276
|
+
imported = 0
|
|
277
|
+
dropped_low = 0
|
|
278
|
+
touched: list[images_mod.Image] = []
|
|
279
|
+
|
|
280
|
+
claimed: set[str] = set()
|
|
281
|
+
for annotation in annotations:
|
|
282
|
+
image = matcher.find(annotation.filename)
|
|
283
|
+
if image is None:
|
|
284
|
+
if annotation.filename not in matcher.ambiguous:
|
|
285
|
+
unmatched_files.append(annotation.filename)
|
|
286
|
+
continue
|
|
287
|
+
if image.id in claimed:
|
|
288
|
+
# Two prediction files resolved to one image. Applying both would
|
|
289
|
+
# let the second silently replace the first.
|
|
290
|
+
matcher.ambiguous.add(annotation.filename)
|
|
291
|
+
continue
|
|
292
|
+
claimed.add(image.id)
|
|
293
|
+
if image.state in PROTECTED_STATES and not overwrite_reviewed:
|
|
294
|
+
skipped_reviewed += 1
|
|
295
|
+
continue
|
|
296
|
+
|
|
297
|
+
keep = []
|
|
298
|
+
for shape in annotation.shapes:
|
|
299
|
+
if shape.confidence is not None and shape.confidence < min_confidence:
|
|
300
|
+
dropped_low += 1
|
|
301
|
+
continue
|
|
302
|
+
keep.append(shape)
|
|
303
|
+
|
|
304
|
+
# Predictions arrive in the source's own coordinate space. If the
|
|
305
|
+
# source did not say how big the image was, trust the project's size --
|
|
306
|
+
# otherwise clamping would squash every box into the top-left corner.
|
|
307
|
+
set_predictions(store, project, image, keep)
|
|
308
|
+
matched += 1
|
|
309
|
+
imported += len(keep)
|
|
310
|
+
touched.append(image)
|
|
311
|
+
|
|
312
|
+
run_id = new_id()
|
|
313
|
+
with store.write() as conn:
|
|
314
|
+
conn.execute(
|
|
315
|
+
"INSERT INTO model_run (id, project_id, name, source, matched, unmatched, created_at)"
|
|
316
|
+
" VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
317
|
+
(
|
|
318
|
+
run_id,
|
|
319
|
+
project.id,
|
|
320
|
+
name or Path(source).name or "model run",
|
|
321
|
+
str(Path(source).name),
|
|
322
|
+
matched,
|
|
323
|
+
len(unmatched_files),
|
|
324
|
+
time.time(),
|
|
325
|
+
),
|
|
326
|
+
)
|
|
327
|
+
projects_mod.touch(conn, project.id)
|
|
328
|
+
|
|
329
|
+
# An attached folder's label files must show what the editor shows.
|
|
330
|
+
# Model shapes are written with their score, so they stay predictions.
|
|
331
|
+
from . import attach
|
|
332
|
+
|
|
333
|
+
for problem in attach.sync(store, project, [image.id for image in touched])[:5]:
|
|
334
|
+
notes.append(f"not written to disk: {problem}")
|
|
335
|
+
|
|
336
|
+
unmatched_images = len(every_image) - matched - skipped_reviewed
|
|
337
|
+
if matcher.ambiguous:
|
|
338
|
+
sample = ", ".join(sorted(matcher.ambiguous)[:5])
|
|
339
|
+
notes.append(
|
|
340
|
+
f"{len(matcher.ambiguous)} prediction file(s) could belong to more than "
|
|
341
|
+
f"one image (several images share the filename) and were not applied, "
|
|
342
|
+
f"for example: {sample}. Keep the folder structure in the predictions "
|
|
343
|
+
f"so each one names its image."
|
|
344
|
+
)
|
|
345
|
+
if unmatched_files:
|
|
346
|
+
notes.append(
|
|
347
|
+
f"{len(unmatched_files)} prediction file(s) matched no image, "
|
|
348
|
+
f"for example: {', '.join(unmatched_files[:5])}"
|
|
349
|
+
)
|
|
350
|
+
if matched == 0:
|
|
351
|
+
notes.append(
|
|
352
|
+
"nothing matched. Predictions are matched to images by path, then "
|
|
353
|
+
"by filename, so check that the names line up."
|
|
354
|
+
)
|
|
355
|
+
if skipped_reviewed:
|
|
356
|
+
notes.append(
|
|
357
|
+
f"{skipped_reviewed} image(s) were already reviewed and were left "
|
|
358
|
+
f"untouched; pass overwrite_reviewed to include them"
|
|
359
|
+
)
|
|
360
|
+
if dropped_low:
|
|
361
|
+
notes.append(f"{dropped_low} prediction(s) below the confidence threshold were dropped")
|
|
362
|
+
|
|
363
|
+
return {
|
|
364
|
+
"run_id": run_id,
|
|
365
|
+
"matched": matched,
|
|
366
|
+
"unmatched_files": len(unmatched_files),
|
|
367
|
+
"ambiguous_files": len(matcher.ambiguous),
|
|
368
|
+
"unmatched_images": max(0, unmatched_images),
|
|
369
|
+
"shapes_imported": imported,
|
|
370
|
+
"skipped_reviewed": skipped_reviewed,
|
|
371
|
+
"dropped_below_threshold": dropped_low,
|
|
372
|
+
"notes": notes,
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def set_predictions(
|
|
377
|
+
store: Store,
|
|
378
|
+
project: projects_mod.Project,
|
|
379
|
+
image: images_mod.Image,
|
|
380
|
+
predicted: list[Any],
|
|
381
|
+
*,
|
|
382
|
+
base_version: int | None = None,
|
|
383
|
+
) -> None:
|
|
384
|
+
"""Replace this image's untouched model shapes with a new run's.
|
|
385
|
+
|
|
386
|
+
Kept: everything a person drew, and every model shape a person has since
|
|
387
|
+
corrected -- that correction is human work now. Cleared: model shapes
|
|
388
|
+
nobody has looked at, which is what stops a second run doubling the boxes.
|
|
389
|
+
|
|
390
|
+
``base_version``: refuse (``Conflict``) if the image changed since the
|
|
391
|
+
caller read it -- a long run must not overwrite what a person did
|
|
392
|
+
meanwhile.
|
|
393
|
+
"""
|
|
394
|
+
existing = shapes_mod.listing(store, project, image)
|
|
395
|
+
keep = [s for s in existing if not s.is_model or s.is_touched]
|
|
396
|
+
|
|
397
|
+
incoming: list[tuple[str | None, Any]] = [(s.id, s.shape) for s in keep]
|
|
398
|
+
incoming.extend((None, shape) for shape in predicted)
|
|
399
|
+
|
|
400
|
+
# advance_state stays off: a prediction is not somebody starting work.
|
|
401
|
+
shapes_mod.save(
|
|
402
|
+
store, project, image, incoming, base_version=base_version,
|
|
403
|
+
origin="model", advance_state=False,
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
# Record what was predicted, so a later deletion still counts as a
|
|
407
|
+
# correction. Without this, removing a false positive would look identical
|
|
408
|
+
# to it never having been predicted.
|
|
409
|
+
kept_model = sum(1 for s in keep if s.is_model)
|
|
410
|
+
with store.write() as conn:
|
|
411
|
+
conn.execute(
|
|
412
|
+
"UPDATE image SET predicted_count = ? WHERE id = ?",
|
|
413
|
+
(len(predicted) + kept_model, image.id),
|
|
414
|
+
)
|
|
415
|
+
|
|
416
|
+
# A reviewed image that has just received fresh, unexamined model shapes is
|
|
417
|
+
# not reviewed any more. Leaving the flag would let the reviewed-only export
|
|
418
|
+
# ship raw predictions as training data -- the self-poisoning that default
|
|
419
|
+
# exists to prevent -- and would count them as agreement besides.
|
|
420
|
+
if image.state == "todo" or (predicted and image.state in PROTECTED_STATES):
|
|
421
|
+
images_mod.set_state(store, project, image, "predicted")
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def accept_all(
|
|
425
|
+
store: Store,
|
|
426
|
+
project: projects_mod.Project,
|
|
427
|
+
image: images_mod.Image,
|
|
428
|
+
) -> int:
|
|
429
|
+
"""Take this image's predictions as correct: mark it reviewed.
|
|
430
|
+
|
|
431
|
+
This is what ``Space`` does, and it is the throughput argument -- on a
|
|
432
|
+
model that is mostly right, a thousand images is fifteen minutes rather
|
|
433
|
+
than a day.
|
|
434
|
+
|
|
435
|
+
Note what it deliberately does *not* do: it leaves ``origin`` alone. The
|
|
436
|
+
first version of this flipped accepted shapes to ``human``, which felt
|
|
437
|
+
tidy and quietly destroyed the agreement metric -- the denominator is
|
|
438
|
+
"model shapes a person has reviewed", and rewriting them out of existence
|
|
439
|
+
means agreement can only ever be computed over shapes nobody has looked at.
|
|
440
|
+
An accepted prediction is still a prediction; what changed is that someone
|
|
441
|
+
has now vouched for it, and that is what the image state records.
|
|
442
|
+
"""
|
|
443
|
+
now = time.time()
|
|
444
|
+
with store.write() as conn:
|
|
445
|
+
confirmed = conn.execute(
|
|
446
|
+
"SELECT COUNT(*) AS n FROM shape WHERE image_id = ? AND origin = 'model'",
|
|
447
|
+
(image.id,),
|
|
448
|
+
).fetchone()["n"]
|
|
449
|
+
conn.execute(
|
|
450
|
+
"UPDATE image SET state = 'reviewed', updated_at = ? WHERE id = ?",
|
|
451
|
+
(now, image.id),
|
|
452
|
+
)
|
|
453
|
+
projects_mod.touch(conn, project.id)
|
|
454
|
+
image.state = "reviewed"
|
|
455
|
+
return int(confirmed)
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def runs(store: Store, project: projects_mod.Project) -> list[dict[str, Any]]:
|
|
459
|
+
return [
|
|
460
|
+
{
|
|
461
|
+
"id": row["id"],
|
|
462
|
+
"name": row["name"],
|
|
463
|
+
"matched": row["matched"],
|
|
464
|
+
"unmatched": row["unmatched"],
|
|
465
|
+
"created_at": row["created_at"],
|
|
466
|
+
}
|
|
467
|
+
for row in store.query(
|
|
468
|
+
"SELECT * FROM model_run WHERE project_id = ? ORDER BY created_at DESC",
|
|
469
|
+
(project.id,),
|
|
470
|
+
)
|
|
471
|
+
]
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
-- Data Factory annotation store -- schema version 1.
|
|
2
|
+
--
|
|
3
|
+
-- One SQLite file per workspace, sitting *beside* the dataset sandboxes and
|
|
4
|
+
-- never inside one, so it is never swept, never zipped by `make_zip`, and
|
|
5
|
+
-- never served. See store.py for why SQLite and not files or Postgres.
|
|
6
|
+
--
|
|
7
|
+
-- Two rules this schema exists to enforce, both of which are correctness
|
|
8
|
+
-- issues rather than conveniences:
|
|
9
|
+
--
|
|
10
|
+
-- 1. A label's index is immutable, never reused and never reordered. Class
|
|
11
|
+
-- indices leak into every YOLO .txt ever exported, so renumbering them
|
|
12
|
+
-- silently re-labels work that is already on disk.
|
|
13
|
+
-- 2. Everything is reachable only through a project, and a project has an
|
|
14
|
+
-- owner. Scoping added later is a rewrite; scoping written now is a JOIN.
|
|
15
|
+
|
|
16
|
+
-- ---------------------------------------------------------------- projects
|
|
17
|
+
|
|
18
|
+
CREATE TABLE IF NOT EXISTS project (
|
|
19
|
+
id TEXT PRIMARY KEY,
|
|
20
|
+
owner_id TEXT NOT NULL DEFAULT 'local',
|
|
21
|
+
name TEXT NOT NULL,
|
|
22
|
+
-- managed: the database is the truth, sidecars are produced by export.
|
|
23
|
+
-- attached: a folder on disk the user opened; every save writes through.
|
|
24
|
+
storage TEXT NOT NULL DEFAULT 'managed',
|
|
25
|
+
-- keep: this is someone's work. ttl: an anonymous upload, swept after a day.
|
|
26
|
+
retention TEXT NOT NULL DEFAULT 'keep',
|
|
27
|
+
-- Only meaningful for attached storage: the folder being annotated.
|
|
28
|
+
root TEXT NOT NULL DEFAULT '',
|
|
29
|
+
-- Monotonic label-index allocator. Deliberately NOT max(idx)+1: retiring
|
|
30
|
+
-- the last label and adding another must not hand out the retired index.
|
|
31
|
+
next_label_idx INTEGER NOT NULL DEFAULT 0,
|
|
32
|
+
created_at REAL NOT NULL,
|
|
33
|
+
updated_at REAL NOT NULL
|
|
34
|
+
);
|
|
35
|
+
|
|
36
|
+
CREATE INDEX IF NOT EXISTS project_owner ON project(owner_id, created_at DESC);
|
|
37
|
+
|
|
38
|
+
-- ------------------------------------------------------------------ labels
|
|
39
|
+
|
|
40
|
+
CREATE TABLE IF NOT EXISTS label (
|
|
41
|
+
project_id TEXT NOT NULL REFERENCES project(id) ON DELETE CASCADE,
|
|
42
|
+
idx INTEGER NOT NULL,
|
|
43
|
+
name TEXT NOT NULL,
|
|
44
|
+
colour TEXT NOT NULL, -- '#rrggbb', assigned server-side
|
|
45
|
+
-- Retiring hides a label from the palette without freeing its index, and
|
|
46
|
+
-- without touching shapes already drawn with it.
|
|
47
|
+
retired_at REAL,
|
|
48
|
+
created_at REAL NOT NULL,
|
|
49
|
+
PRIMARY KEY (project_id, idx)
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
-- Names are unique among *live* labels only, so a retired name can come back
|
|
53
|
+
-- later -- as a new index, which is the whole point.
|
|
54
|
+
CREATE UNIQUE INDEX IF NOT EXISTS label_live_name
|
|
55
|
+
ON label(project_id, name) WHERE retired_at IS NULL;
|
|
56
|
+
|
|
57
|
+
-- ------------------------------------------------------------------ images
|
|
58
|
+
|
|
59
|
+
CREATE TABLE IF NOT EXISTS image (
|
|
60
|
+
id TEXT PRIMARY KEY,
|
|
61
|
+
project_id TEXT NOT NULL REFERENCES project(id) ON DELETE CASCADE,
|
|
62
|
+
-- POSIX-relative to the project root, so a database made on Windows opens
|
|
63
|
+
-- on Linux. Never absolute: an absolute path in a row is a path a client
|
|
64
|
+
-- could eventually influence.
|
|
65
|
+
rel_path TEXT NOT NULL,
|
|
66
|
+
-- Oriented dimensions -- what the pixels actually are after EXIF is
|
|
67
|
+
-- applied, which is what the browser draws and what boxes are relative to.
|
|
68
|
+
width INTEGER NOT NULL DEFAULT 0,
|
|
69
|
+
height INTEGER NOT NULL DEFAULT 0,
|
|
70
|
+
seq INTEGER NOT NULL DEFAULT 0,
|
|
71
|
+
-- todo -> predicted -> in_progress -> reviewed, plus skipped ("I can't
|
|
72
|
+
-- tell"), which must never quietly become training data.
|
|
73
|
+
state TEXT NOT NULL DEFAULT 'todo',
|
|
74
|
+
-- Bumped on every save. A save carries the version it was based on, so a
|
|
75
|
+
-- second tab editing the same image gets a 409 instead of clobbering.
|
|
76
|
+
version INTEGER NOT NULL DEFAULT 0,
|
|
77
|
+
created_at REAL NOT NULL,
|
|
78
|
+
updated_at REAL NOT NULL
|
|
79
|
+
);
|
|
80
|
+
|
|
81
|
+
CREATE UNIQUE INDEX IF NOT EXISTS image_path ON image(project_id, rel_path);
|
|
82
|
+
CREATE INDEX IF NOT EXISTS image_seq ON image(project_id, seq);
|
|
83
|
+
-- "the next unreviewed image" is the single hottest query in the editor.
|
|
84
|
+
CREATE INDEX IF NOT EXISTS image_state ON image(project_id, state, seq);
|
|
85
|
+
|
|
86
|
+
-- ------------------------------------------------------------------ shapes
|
|
87
|
+
|
|
88
|
+
CREATE TABLE IF NOT EXISTS shape (
|
|
89
|
+
id TEXT PRIMARY KEY,
|
|
90
|
+
image_id TEXT NOT NULL REFERENCES image(id) ON DELETE CASCADE,
|
|
91
|
+
project_id TEXT NOT NULL,
|
|
92
|
+
label_idx INTEGER NOT NULL,
|
|
93
|
+
-- Box columns are real columns, not JSON, because size distribution and
|
|
94
|
+
-- geometry-error analysis are queries, not scans.
|
|
95
|
+
xmin INTEGER NOT NULL,
|
|
96
|
+
ymin INTEGER NOT NULL,
|
|
97
|
+
xmax INTEGER NOT NULL,
|
|
98
|
+
ymax INTEGER NOT NULL,
|
|
99
|
+
polygon TEXT, -- JSON [[x, y], ...], or NULL
|
|
100
|
+
-- human | model. Model shapes render dashed until someone touches them.
|
|
101
|
+
origin TEXT NOT NULL DEFAULT 'human',
|
|
102
|
+
confidence REAL,
|
|
103
|
+
pose TEXT NOT NULL DEFAULT 'Unspecified',
|
|
104
|
+
truncated INTEGER NOT NULL DEFAULT 0,
|
|
105
|
+
difficult INTEGER NOT NULL DEFAULT 0,
|
|
106
|
+
occluded INTEGER NOT NULL DEFAULT 0,
|
|
107
|
+
created_at REAL NOT NULL,
|
|
108
|
+
-- Set when a person edits a model shape. That is what turns it solid, and
|
|
109
|
+
-- what the agreement metric counts.
|
|
110
|
+
edited_at REAL,
|
|
111
|
+
FOREIGN KEY (project_id, label_idx) REFERENCES label(project_id, idx)
|
|
112
|
+
);
|
|
113
|
+
|
|
114
|
+
CREATE INDEX IF NOT EXISTS shape_image ON shape(image_id);
|
|
115
|
+
CREATE INDEX IF NOT EXISTS shape_label ON shape(project_id, label_idx);
|
|
116
|
+
|
|
117
|
+
-- -------------------------------------------------------------- model runs
|
|
118
|
+
|
|
119
|
+
CREATE TABLE IF NOT EXISTS model_run (
|
|
120
|
+
id TEXT PRIMARY KEY,
|
|
121
|
+
project_id TEXT NOT NULL REFERENCES project(id) ON DELETE CASCADE,
|
|
122
|
+
name TEXT NOT NULL,
|
|
123
|
+
source TEXT NOT NULL DEFAULT '',
|
|
124
|
+
matched INTEGER NOT NULL DEFAULT 0,
|
|
125
|
+
unmatched INTEGER NOT NULL DEFAULT 0,
|
|
126
|
+
created_at REAL NOT NULL
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
CREATE INDEX IF NOT EXISTS model_run_project ON model_run(project_id, created_at DESC);
|
|
130
|
+
|
|
131
|
+
-- ------------------------------------------------------------------- audit
|
|
132
|
+
|
|
133
|
+
CREATE TABLE IF NOT EXISTS audit (
|
|
134
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
135
|
+
project_id TEXT NOT NULL,
|
|
136
|
+
owner_id TEXT NOT NULL,
|
|
137
|
+
action TEXT NOT NULL,
|
|
138
|
+
detail TEXT NOT NULL DEFAULT '',
|
|
139
|
+
at REAL NOT NULL
|
|
140
|
+
);
|
|
141
|
+
|
|
142
|
+
CREATE INDEX IF NOT EXISTS audit_project ON audit(project_id, at DESC);
|
|
143
|
+
|
|
144
|
+
-- --------------------------------------------------- accounts (Phase 5)
|
|
145
|
+
--
|
|
146
|
+
-- Created now, unused now. They cost nothing empty, and having the tables
|
|
147
|
+
-- present means Phase 5 adds routes rather than a migration across live data.
|
|
148
|
+
|
|
149
|
+
CREATE TABLE IF NOT EXISTS user (
|
|
150
|
+
id TEXT PRIMARY KEY,
|
|
151
|
+
email TEXT NOT NULL,
|
|
152
|
+
password_hash TEXT NOT NULL DEFAULT '',
|
|
153
|
+
created_at REAL NOT NULL
|
|
154
|
+
);
|
|
155
|
+
|
|
156
|
+
CREATE UNIQUE INDEX IF NOT EXISTS user_email ON user(email);
|
|
157
|
+
|
|
158
|
+
CREATE TABLE IF NOT EXISTS session (
|
|
159
|
+
token TEXT PRIMARY KEY,
|
|
160
|
+
user_id TEXT NOT NULL REFERENCES user(id) ON DELETE CASCADE,
|
|
161
|
+
created_at REAL NOT NULL,
|
|
162
|
+
expires_at REAL NOT NULL
|
|
163
|
+
);
|
|
164
|
+
|
|
165
|
+
CREATE TABLE IF NOT EXISTS member (
|
|
166
|
+
project_id TEXT NOT NULL REFERENCES project(id) ON DELETE CASCADE,
|
|
167
|
+
user_id TEXT NOT NULL REFERENCES user(id) ON DELETE CASCADE,
|
|
168
|
+
role TEXT NOT NULL DEFAULT 'owner',
|
|
169
|
+
created_at REAL NOT NULL,
|
|
170
|
+
PRIMARY KEY (project_id, user_id)
|
|
171
|
+
);
|