easyannotate 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_factory/__init__.py +82 -0
- data_factory/__main__.py +6 -0
- data_factory/annotate/__init__.py +15 -0
- data_factory/annotate/attach.py +398 -0
- data_factory/annotate/demo.py +161 -0
- data_factory/annotate/images.py +443 -0
- data_factory/annotate/labels.py +399 -0
- data_factory/annotate/loop.py +359 -0
- data_factory/annotate/projects.py +207 -0
- data_factory/annotate/review.py +471 -0
- data_factory/annotate/schema.sql +171 -0
- data_factory/annotate/shapes.py +488 -0
- data_factory/annotate/store.py +292 -0
- data_factory/annotate/tools/__init__.py +39 -0
- data_factory/annotate/tools/fake.py +170 -0
- data_factory/annotate/tools/install.py +254 -0
- data_factory/annotate/tools/local.py +389 -0
- data_factory/annotate/tools/postprocess.py +175 -0
- data_factory/annotate/tools/protocol.py +238 -0
- data_factory/annotate/tools/registry.py +196 -0
- data_factory/annotate/tools/results.py +83 -0
- data_factory/annotate/tools/serve.py +106 -0
- data_factory/cli/__init__.py +21 -0
- data_factory/cli/main.py +837 -0
- data_factory/cli/menu.py +173 -0
- data_factory/cli/render.py +158 -0
- data_factory/core/__init__.py +11 -0
- data_factory/core/annotations/__init__.py +5 -0
- data_factory/core/annotations/coco.py +394 -0
- data_factory/core/annotations/csvfmt.py +174 -0
- data_factory/core/annotations/dota.py +98 -0
- data_factory/core/annotations/formats.py +626 -0
- data_factory/core/annotations/labelme.py +153 -0
- data_factory/core/annotations/model.py +334 -0
- data_factory/core/annotations/voc.py +231 -0
- data_factory/core/annotations/yolo.py +283 -0
- data_factory/core/bktree.py +89 -0
- data_factory/core/fs.py +245 -0
- data_factory/core/imaging.py +626 -0
- data_factory/core/ops/__init__.py +44 -0
- data_factory/core/ops/analyze.py +383 -0
- data_factory/core/ops/annotate_export.py +326 -0
- data_factory/core/ops/audit.py +274 -0
- data_factory/core/ops/augment.py +454 -0
- data_factory/core/ops/collect.py +241 -0
- data_factory/core/ops/combine.py +582 -0
- data_factory/core/ops/convert.py +212 -0
- data_factory/core/ops/dataset.py +246 -0
- data_factory/core/ops/dedup.py +235 -0
- data_factory/core/ops/export.py +168 -0
- data_factory/core/ops/image.py +407 -0
- data_factory/core/ops/labels.py +215 -0
- data_factory/core/ops/pairing.py +133 -0
- data_factory/core/ops/rename.py +154 -0
- data_factory/core/ops/report.py +323 -0
- data_factory/core/ops/visualize.py +164 -0
- data_factory/core/pairs.py +144 -0
- data_factory/core/parallel.py +191 -0
- data_factory/core/provenance.py +136 -0
- data_factory/core/registry.py +339 -0
- data_factory/core/stats.py +403 -0
- data_factory/core/types.py +148 -0
- data_factory/core/writer.py +194 -0
- data_factory/desktop.py +207 -0
- data_factory/hardware.py +243 -0
- data_factory/pipeline.py +192 -0
- data_factory/py.typed +0 -0
- data_factory/service/__init__.py +26 -0
- data_factory/service/app.py +483 -0
- data_factory/service/binding.py +174 -0
- data_factory/service/deps.py +133 -0
- data_factory/service/jobs.py +177 -0
- data_factory/service/limits.py +71 -0
- data_factory/service/routers/__init__.py +11 -0
- data_factory/service/routers/annotate.py +582 -0
- data_factory/service/routers/local.py +464 -0
- data_factory/service/routers/media.py +154 -0
- data_factory/service/routers/predictions.py +116 -0
- data_factory/service/routers/promote.py +182 -0
- data_factory/service/routers/tools.py +308 -0
- data_factory/service/static/annotate/api.js +246 -0
- data_factory/service/static/annotate/app.js +1696 -0
- data_factory/service/static/annotate/assist.js +106 -0
- data_factory/service/static/annotate/filmstrip.js +286 -0
- data_factory/service/static/annotate/hit.js +146 -0
- data_factory/service/static/annotate/labels.js +197 -0
- data_factory/service/static/annotate/render.js +286 -0
- data_factory/service/static/annotate/scene.js +277 -0
- data_factory/service/static/annotate/shapelist.js +72 -0
- data_factory/service/static/annotate/tools/box.js +96 -0
- data_factory/service/static/annotate/tools/line.js +96 -0
- data_factory/service/static/annotate/tools/obb.js +88 -0
- data_factory/service/static/annotate/tools/point.js +49 -0
- data_factory/service/static/annotate/tools/polygon.js +110 -0
- data_factory/service/static/annotate/tools/select.js +227 -0
- data_factory/service/static/annotate/undo.js +176 -0
- data_factory/service/static/annotate/viewport.js +156 -0
- data_factory/service/static/annotate.html +390 -0
- data_factory/service/static/index.html +632 -0
- data_factory/service/workspace.py +244 -0
- easyannotate/__init__.py +64 -0
- easyannotate/__main__.py +5 -0
- easyannotate/project.py +181 -0
- easyannotate/py.typed +0 -0
- easyannotate-0.4.0.dist-info/METADATA +767 -0
- easyannotate-0.4.0.dist-info/RECORD +109 -0
- easyannotate-0.4.0.dist-info/WHEEL +4 -0
- easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
- easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
"""The loop: reviewed images make a model, the model predicts the rest, and a
|
|
2
|
+
person reviews its mistakes -- hardest first.
|
|
3
|
+
|
|
4
|
+
easyannotate train ./doors # reviewed -> model -> predictions
|
|
5
|
+
easyannotate predict ./doors --model best.pt
|
|
6
|
+
|
|
7
|
+
**No client names a model file.** A ``.pt`` file is a pickle, and loading a
|
|
8
|
+
model runs it. On the command line the path comes from a person typing it
|
|
9
|
+
into their own terminal. The app's Train/Predict buttons (the token-fenced
|
|
10
|
+
local routes) use only the project's own latest model, found here under
|
|
11
|
+
``models_home()`` -- the page sends a project id, never a path.
|
|
12
|
+
|
|
13
|
+
Training and inference run in the tools environment (``easyannotate tools
|
|
14
|
+
install``), never in this one: inference through the sidecar protocol, and
|
|
15
|
+
training as a subprocess of the tools interpreter, so torch is never imported
|
|
16
|
+
here and a crashed training run takes nothing else down with it.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
import shutil
|
|
24
|
+
import subprocess
|
|
25
|
+
import sys
|
|
26
|
+
import tempfile
|
|
27
|
+
import time
|
|
28
|
+
from collections.abc import Callable, Iterable
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import Any
|
|
31
|
+
|
|
32
|
+
from . import attach as attach_mod
|
|
33
|
+
from . import images as images_mod
|
|
34
|
+
from . import labels as labels_mod
|
|
35
|
+
from . import review
|
|
36
|
+
from . import shapes as shapes_mod
|
|
37
|
+
from .projects import Project
|
|
38
|
+
from .store import Store, StoreError
|
|
39
|
+
from .tools.install import read_profile
|
|
40
|
+
from .tools.local import child_environment, python_for_tools
|
|
41
|
+
from .tools.registry import REGISTRY
|
|
42
|
+
from .tools.results import shapes_from_result
|
|
43
|
+
|
|
44
|
+
MODEL_TOOL = "detect.model"
|
|
45
|
+
#: The module the tools interpreter runs to train. Overridable so the test
|
|
46
|
+
#: suite can stand in a trainer that needs no torch.
|
|
47
|
+
TRAIN_MODULE_ENV = "EASYANNOTATE_TRAIN_MODULE"
|
|
48
|
+
DEFAULT_TRAIN_MODULE = "easyannotate_tools.train"
|
|
49
|
+
#: Lines the trainer prints for us, among Ultralytics' own.
|
|
50
|
+
EPOCH_PREFIX = "EASYANNOTATE_EPOCH "
|
|
51
|
+
RESULT_PREFIX = "EASYANNOTATE_RESULT "
|
|
52
|
+
#: A newer model replaces an older one's untouched guesses on "predicted"
|
|
53
|
+
#: images too -- that is what each turn of the loop is for.
|
|
54
|
+
#: Not "in_progress": someone is working on those, and predictions dropped on
|
|
55
|
+
#: top of their boxes would not even appear in the review queue.
|
|
56
|
+
UNREVIEWED = ("todo", "predicted")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class LoopError(StoreError):
|
|
60
|
+
"""Something a person can act on: say what, and what to do."""
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def models_home() -> Path:
|
|
64
|
+
"""Where trained models and training runs are kept. Shared with the
|
|
65
|
+
sidecar, which downloads base weights here too."""
|
|
66
|
+
path = Path(os.environ.get("EASYANNOTATE_MODELS") or Path.home() / ".datafactory" / "models")
|
|
67
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
68
|
+
return path
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# ---------------------------------------------------------------- predict
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def predict(
|
|
75
|
+
store: Store,
|
|
76
|
+
project: Project,
|
|
77
|
+
image_root: Path,
|
|
78
|
+
model: Path,
|
|
79
|
+
*,
|
|
80
|
+
conf: float = 0.25,
|
|
81
|
+
states: Iterable[str] = UNREVIEWED,
|
|
82
|
+
tool: str = MODEL_TOOL,
|
|
83
|
+
on_progress: Callable[[int, int, str], None] | None = None,
|
|
84
|
+
) -> dict[str, Any]:
|
|
85
|
+
"""Run a trained model over the project's unreviewed images.
|
|
86
|
+
|
|
87
|
+
Its answers go in through ``set_predictions`` -- the same path as an
|
|
88
|
+
imported run -- so they are dashed, scored, ordered by uncertainty, and
|
|
89
|
+
never exported until somebody has looked at them. Reviewed images are
|
|
90
|
+
never touched: finished work is not something a model gets to redo.
|
|
91
|
+
"""
|
|
92
|
+
model = Path(model).expanduser().resolve()
|
|
93
|
+
if not model.is_file():
|
|
94
|
+
raise LoopError(f"no model file at {model}")
|
|
95
|
+
if REGISTRY.spec(tool) is None:
|
|
96
|
+
raise LoopError(
|
|
97
|
+
"no inference tool is installed -- run: easyannotate tools install"
|
|
98
|
+
)
|
|
99
|
+
wanted = [s for s in states if s in images_mod.STATES and s != "reviewed"]
|
|
100
|
+
if not wanted:
|
|
101
|
+
raise LoopError("predict runs on unreviewed images only")
|
|
102
|
+
|
|
103
|
+
targets: list[images_mod.Image] = []
|
|
104
|
+
for state in wanted:
|
|
105
|
+
offset = 0
|
|
106
|
+
while True:
|
|
107
|
+
page = images_mod.listing(store, project, state=state, limit=5000, offset=offset)
|
|
108
|
+
targets.extend(page)
|
|
109
|
+
if len(page) < 5000:
|
|
110
|
+
break
|
|
111
|
+
offset += 5000
|
|
112
|
+
|
|
113
|
+
predicted = shapes_total = failed = skipped = 0
|
|
114
|
+
first_error = ""
|
|
115
|
+
touched: list[str] = []
|
|
116
|
+
started = time.time()
|
|
117
|
+
root = Path(image_root).resolve()
|
|
118
|
+
try:
|
|
119
|
+
for index, image in enumerate(targets):
|
|
120
|
+
if on_progress:
|
|
121
|
+
on_progress(index, len(targets), image.name)
|
|
122
|
+
path = (root / image.rel_path).resolve()
|
|
123
|
+
if not path.is_relative_to(root) or not path.is_file():
|
|
124
|
+
failed += 1
|
|
125
|
+
continue
|
|
126
|
+
try:
|
|
127
|
+
raw = REGISTRY.invoke_waiting(
|
|
128
|
+
tool,
|
|
129
|
+
{
|
|
130
|
+
"image_id": image.id,
|
|
131
|
+
"image_path": str(path),
|
|
132
|
+
"width": image.width,
|
|
133
|
+
"height": image.height,
|
|
134
|
+
"model": str(model),
|
|
135
|
+
"conf": conf,
|
|
136
|
+
},
|
|
137
|
+
budget_s=600.0,
|
|
138
|
+
)
|
|
139
|
+
except Exception as exc:
|
|
140
|
+
failed += 1
|
|
141
|
+
if not first_error:
|
|
142
|
+
first_error = f"{image.name}: {exc}"
|
|
143
|
+
continue
|
|
144
|
+
made = shapes_from_result(raw, "", image.width, image.height)
|
|
145
|
+
# The list was read minutes ago. Someone reviewing alongside the
|
|
146
|
+
# run may have accepted or started this image since: re-read it,
|
|
147
|
+
# and refuse to write over anything that moved on.
|
|
148
|
+
try:
|
|
149
|
+
current = images_mod.get(store, project, image.id)
|
|
150
|
+
if current.state not in wanted:
|
|
151
|
+
skipped += 1
|
|
152
|
+
continue
|
|
153
|
+
review.set_predictions(store, project, current, made, base_version=current.version)
|
|
154
|
+
except StoreError:
|
|
155
|
+
skipped += 1
|
|
156
|
+
continue
|
|
157
|
+
touched.append(image.id)
|
|
158
|
+
predicted += 1
|
|
159
|
+
shapes_total += len(made)
|
|
160
|
+
if on_progress:
|
|
161
|
+
on_progress(len(targets), len(targets), "done")
|
|
162
|
+
finally:
|
|
163
|
+
# Even when cancelled halfway: what was predicted is in the database,
|
|
164
|
+
# so it goes to the folder's label files too.
|
|
165
|
+
attach_mod.sync(store, project, touched)
|
|
166
|
+
return {
|
|
167
|
+
"images": len(targets),
|
|
168
|
+
"predicted": predicted,
|
|
169
|
+
"shapes": shapes_total,
|
|
170
|
+
"failed": failed,
|
|
171
|
+
"skipped_changed": skipped,
|
|
172
|
+
"first_error": first_error,
|
|
173
|
+
"seconds": round(time.time() - started, 1),
|
|
174
|
+
"agreement": shapes_mod.agreement(store, project),
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
# ------------------------------------------------------------------ train
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def train(
|
|
182
|
+
store: Store,
|
|
183
|
+
project: Project,
|
|
184
|
+
*,
|
|
185
|
+
epochs: int = 30,
|
|
186
|
+
imgsz: int = 640,
|
|
187
|
+
base: str = "yolov8n.pt",
|
|
188
|
+
on_line: Callable[[str], None] = print,
|
|
189
|
+
) -> dict[str, Any]:
|
|
190
|
+
"""Train a detector on the reviewed images, in the tools environment.
|
|
191
|
+
|
|
192
|
+
Exports exactly what export would -- reviewed images, the project's own
|
|
193
|
+
class order, so index 3 means in the model what it means in the files --
|
|
194
|
+
and hands the folder to Ultralytics. Returns the saved model's path and
|
|
195
|
+
its final metrics.
|
|
196
|
+
"""
|
|
197
|
+
reviewed = images_mod.counts(store, project).get("reviewed", 0)
|
|
198
|
+
if reviewed == 0:
|
|
199
|
+
raise LoopError("nothing to learn from yet -- review some images first")
|
|
200
|
+
classes = labels_mod.class_list(store, project)
|
|
201
|
+
if not classes:
|
|
202
|
+
raise LoopError("the project has no classes yet")
|
|
203
|
+
|
|
204
|
+
python = python_for_tools()
|
|
205
|
+
module = os.environ.get(TRAIN_MODULE_ENV) or DEFAULT_TRAIN_MODULE
|
|
206
|
+
if module == DEFAULT_TRAIN_MODULE and not _can_import(python, "ultralytics"):
|
|
207
|
+
raise LoopError("training needs the tools -- run: easyannotate tools install")
|
|
208
|
+
|
|
209
|
+
stamp = time.strftime("%Y%m%d-%H%M%S")
|
|
210
|
+
runs = models_home() / "runs" / project.id
|
|
211
|
+
runs.mkdir(parents=True, exist_ok=True)
|
|
212
|
+
log_path = runs / f"{stamp}.log"
|
|
213
|
+
|
|
214
|
+
with tempfile.TemporaryDirectory(prefix="easyannotate-train-") as holding:
|
|
215
|
+
data = Path(holding) / "data"
|
|
216
|
+
_export_reviewed(store, project, data)
|
|
217
|
+
yaml_path = _write_training_yaml(data, classes)
|
|
218
|
+
on_line(f"training on {reviewed} reviewed image(s), {len(classes)} class(es), {epochs} epoch(s)")
|
|
219
|
+
|
|
220
|
+
command = [
|
|
221
|
+
python, "-m", module,
|
|
222
|
+
"--data", str(yaml_path),
|
|
223
|
+
"--model", base,
|
|
224
|
+
"--epochs", str(int(epochs)),
|
|
225
|
+
"--imgsz", str(int(imgsz)),
|
|
226
|
+
"--out", str(runs),
|
|
227
|
+
"--name", stamp,
|
|
228
|
+
]
|
|
229
|
+
device = _train_device()
|
|
230
|
+
if device:
|
|
231
|
+
# What the install verified, not torch.cuda.is_available(): that
|
|
232
|
+
# says yes on a driver the CUDA build cannot actually run on.
|
|
233
|
+
command += ["--device", device]
|
|
234
|
+
result: dict[str, Any] | None = None
|
|
235
|
+
tail: list[str] = []
|
|
236
|
+
env = child_environment(PYTHONUNBUFFERED="1")
|
|
237
|
+
with open(log_path, "w", encoding="utf-8") as log:
|
|
238
|
+
process = subprocess.Popen(
|
|
239
|
+
command,
|
|
240
|
+
stdout=subprocess.PIPE,
|
|
241
|
+
stderr=subprocess.STDOUT,
|
|
242
|
+
text=True,
|
|
243
|
+
encoding="utf-8",
|
|
244
|
+
errors="replace",
|
|
245
|
+
cwd=models_home(),
|
|
246
|
+
env=env,
|
|
247
|
+
)
|
|
248
|
+
assert process.stdout is not None
|
|
249
|
+
try:
|
|
250
|
+
for line in process.stdout:
|
|
251
|
+
log.write(line)
|
|
252
|
+
line = line.rstrip()
|
|
253
|
+
tail = (tail + [line])[-15:]
|
|
254
|
+
if line.startswith(EPOCH_PREFIX):
|
|
255
|
+
on_line(line[len(EPOCH_PREFIX):])
|
|
256
|
+
elif line.startswith(RESULT_PREFIX):
|
|
257
|
+
try:
|
|
258
|
+
result = json.loads(line[len(RESULT_PREFIX):])
|
|
259
|
+
except ValueError:
|
|
260
|
+
result = None
|
|
261
|
+
code = process.wait()
|
|
262
|
+
finally:
|
|
263
|
+
# Never leave a trainer running behind us -- the temporary
|
|
264
|
+
# dataset it reads is about to be deleted.
|
|
265
|
+
if process.poll() is None:
|
|
266
|
+
process.kill()
|
|
267
|
+
process.wait()
|
|
268
|
+
if code != 0 or result is None:
|
|
269
|
+
raise LoopError(
|
|
270
|
+
f"training failed (exit {code}); full log: {log_path}\n" + "\n".join(tail[-8:])
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
best = Path(result["best"])
|
|
274
|
+
if not best.is_file():
|
|
275
|
+
raise LoopError(f"training reported {best}, which does not exist; log: {log_path}")
|
|
276
|
+
saved = models_home() / "projects" / project.id / f"{stamp}.pt"
|
|
277
|
+
saved.parent.mkdir(parents=True, exist_ok=True)
|
|
278
|
+
shutil.copy2(best, saved)
|
|
279
|
+
return {"model": saved, "metrics": result.get("metrics") or {}, "log": log_path}
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _train_device() -> str:
|
|
283
|
+
"""The device the install verified; empty -- the trainer decides -- with
|
|
284
|
+
no profile, e.g. a tools Python named by DATAFACTORY_TOOLS_PYTHON."""
|
|
285
|
+
device = read_profile().get("device", "")
|
|
286
|
+
if device == "cuda":
|
|
287
|
+
return str(int(read_profile().get("gpu_index") or 0))
|
|
288
|
+
return device if device in ("mps", "cpu") else ""
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def latest_model(project: Project) -> Path | None:
|
|
292
|
+
folder = models_home() / "projects" / project.id
|
|
293
|
+
found = sorted(folder.glob("*.pt")) if folder.is_dir() else []
|
|
294
|
+
return found[-1] if found else None
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _export_reviewed(store: Store, project: Project, output: Path) -> None:
|
|
298
|
+
from ..core import registry
|
|
299
|
+
from ..core.types import Context
|
|
300
|
+
|
|
301
|
+
result = registry.get("annotate.export").run(
|
|
302
|
+
{
|
|
303
|
+
"database": str(store.path),
|
|
304
|
+
"project": project.id,
|
|
305
|
+
"output": str(output),
|
|
306
|
+
"to": "yolo",
|
|
307
|
+
"states": ["reviewed"],
|
|
308
|
+
"owner": project.owner_id,
|
|
309
|
+
"copy_images": True,
|
|
310
|
+
# Boxes: a detector learns from boxes, and a polygon line in a
|
|
311
|
+
# detection label file is a guess about what the trainer does.
|
|
312
|
+
"segmentation": False,
|
|
313
|
+
"include_empty": True,
|
|
314
|
+
},
|
|
315
|
+
Context(),
|
|
316
|
+
)
|
|
317
|
+
if not result.ok:
|
|
318
|
+
raise LoopError("could not export the reviewed images: " + "; ".join(result.errors))
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _write_training_yaml(data: Path, classes: list[str]) -> Path:
|
|
322
|
+
import yaml
|
|
323
|
+
|
|
324
|
+
images = data / "images"
|
|
325
|
+
splits = sorted(p.name for p in images.iterdir() if p.is_dir()) if images.is_dir() else []
|
|
326
|
+
train_split = "train" if "train" in splits else (splits[0] if splits else "train")
|
|
327
|
+
# A small project has no validation split. Validating on the training
|
|
328
|
+
# images says nothing about generalisation, but it lets a first model
|
|
329
|
+
# exist -- and the review queue, not mAP, is what judges it here.
|
|
330
|
+
val_split = "val" if "val" in splits else train_split
|
|
331
|
+
path = data / "train.yaml"
|
|
332
|
+
path.write_text(
|
|
333
|
+
yaml.safe_dump(
|
|
334
|
+
{
|
|
335
|
+
"path": str(data),
|
|
336
|
+
"train": f"images/{train_split}",
|
|
337
|
+
"val": f"images/{val_split}",
|
|
338
|
+
"names": dict(enumerate(classes)),
|
|
339
|
+
},
|
|
340
|
+
sort_keys=False,
|
|
341
|
+
),
|
|
342
|
+
encoding="utf-8",
|
|
343
|
+
)
|
|
344
|
+
return path
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def _can_import(python: str, module: str) -> bool:
|
|
348
|
+
if python == sys.executable:
|
|
349
|
+
try:
|
|
350
|
+
__import__(module)
|
|
351
|
+
return True
|
|
352
|
+
except ImportError:
|
|
353
|
+
return False
|
|
354
|
+
try:
|
|
355
|
+
return subprocess.run(
|
|
356
|
+
[python, "-c", f"import {module}"], capture_output=True, timeout=120, env=child_environment()
|
|
357
|
+
).returncode == 0
|
|
358
|
+
except Exception:
|
|
359
|
+
return False
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""Projects: the unit of ownership, and the gate everything else passes.
|
|
2
|
+
|
|
3
|
+
There is one structural rule here worth stating plainly, because it is what
|
|
4
|
+
makes the multi-tenant story in Phase 5 additive rather than a rewrite:
|
|
5
|
+
|
|
6
|
+
**Nothing below a project is reachable without first resolving the project
|
|
7
|
+
for an owner.** ``labels``, ``images`` and ``shapes`` all take a
|
|
8
|
+
:class:`Project`, and the only way to get one is :func:`get`, which filters
|
|
9
|
+
by ``owner_id``.
|
|
10
|
+
|
|
11
|
+
That is deliberately stronger than "remember to add ``WHERE owner_id = ?`` to
|
|
12
|
+
every query". A forgotten WHERE clause is a data leak that no test will notice;
|
|
13
|
+
a missing function argument does not compile.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import os
|
|
19
|
+
import time
|
|
20
|
+
from dataclasses import dataclass
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
from .store import LOCAL_OWNER, NotFound, Store, StoreError, new_id, require_id
|
|
24
|
+
|
|
25
|
+
#: managed -- the database is the truth, sidecar files come out of export.
|
|
26
|
+
#: attached -- a folder the user opened; every save is written through to the
|
|
27
|
+
#: sidecar on disk, which is what "my XML gets updated" means.
|
|
28
|
+
STORAGE_MODES = ("managed", "attached")
|
|
29
|
+
|
|
30
|
+
RETENTIONS = ("keep", "ttl")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def attached_allowed() -> bool:
|
|
34
|
+
"""Attached storage is a local-only mode.
|
|
35
|
+
|
|
36
|
+
On a host, a project pointing at a server-side folder would be exactly the
|
|
37
|
+
arbitrary-path primitive the whole service layer exists to refuse. Hosted
|
|
38
|
+
deployments set ``DATAFACTORY_ALLOW_ATTACHED=0``.
|
|
39
|
+
"""
|
|
40
|
+
setting = os.environ.get("DATAFACTORY_ALLOW_ATTACHED", "1").strip().lower()
|
|
41
|
+
return setting not in ("0", "false", "no", "off")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(slots=True)
|
|
45
|
+
class Project:
|
|
46
|
+
id: str
|
|
47
|
+
owner_id: str
|
|
48
|
+
name: str
|
|
49
|
+
storage: str
|
|
50
|
+
retention: str
|
|
51
|
+
root: str
|
|
52
|
+
next_label_idx: int
|
|
53
|
+
created_at: float
|
|
54
|
+
updated_at: float
|
|
55
|
+
#: Which per-image sidecar format an attached project writes back to:
|
|
56
|
+
#: 'voc', 'labelme', 'yolo', or '' for none. Empty for managed projects,
|
|
57
|
+
#: and empty for attached folders in a collected format -- see attach.py.
|
|
58
|
+
sidecar_format: str = ""
|
|
59
|
+
|
|
60
|
+
@property
|
|
61
|
+
def is_attached(self) -> bool:
|
|
62
|
+
return self.storage == "attached"
|
|
63
|
+
|
|
64
|
+
def as_dict(self) -> dict[str, Any]:
|
|
65
|
+
return {
|
|
66
|
+
"id": self.id,
|
|
67
|
+
"name": self.name,
|
|
68
|
+
"storage": self.storage,
|
|
69
|
+
"retention": self.retention,
|
|
70
|
+
"sidecar_format": self.sidecar_format,
|
|
71
|
+
"created_at": self.created_at,
|
|
72
|
+
"updated_at": self.updated_at,
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _row_to_project(row: Any) -> Project:
|
|
77
|
+
return Project(
|
|
78
|
+
id=row["id"],
|
|
79
|
+
owner_id=row["owner_id"],
|
|
80
|
+
name=row["name"],
|
|
81
|
+
storage=row["storage"],
|
|
82
|
+
retention=row["retention"],
|
|
83
|
+
root=row["root"],
|
|
84
|
+
next_label_idx=row["next_label_idx"],
|
|
85
|
+
created_at=row["created_at"],
|
|
86
|
+
updated_at=row["updated_at"],
|
|
87
|
+
sidecar_format=row["sidecar_format"],
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def create(
|
|
92
|
+
store: Store,
|
|
93
|
+
name: str,
|
|
94
|
+
*,
|
|
95
|
+
owner_id: str = LOCAL_OWNER,
|
|
96
|
+
storage: str = "managed",
|
|
97
|
+
retention: str = "keep",
|
|
98
|
+
root: str = "",
|
|
99
|
+
sidecar_format: str = "",
|
|
100
|
+
) -> Project:
|
|
101
|
+
"""Start a project. No labels, no schema step -- that is the whole point."""
|
|
102
|
+
clean = (name or "").strip() or "Untitled project"
|
|
103
|
+
if storage not in STORAGE_MODES:
|
|
104
|
+
raise StoreError(f"storage must be one of {', '.join(STORAGE_MODES)}")
|
|
105
|
+
if retention not in RETENTIONS:
|
|
106
|
+
raise StoreError(f"retention must be one of {', '.join(RETENTIONS)}")
|
|
107
|
+
if storage == "attached" and not attached_allowed():
|
|
108
|
+
raise StoreError("attached projects are disabled on this deployment")
|
|
109
|
+
|
|
110
|
+
now = time.time()
|
|
111
|
+
project = Project(
|
|
112
|
+
id=new_id(),
|
|
113
|
+
owner_id=owner_id,
|
|
114
|
+
name=clean,
|
|
115
|
+
storage=storage,
|
|
116
|
+
retention=retention,
|
|
117
|
+
root=root if storage == "attached" else "",
|
|
118
|
+
next_label_idx=0,
|
|
119
|
+
created_at=now,
|
|
120
|
+
updated_at=now,
|
|
121
|
+
sidecar_format=sidecar_format if storage == "attached" else "",
|
|
122
|
+
)
|
|
123
|
+
with store.write() as conn:
|
|
124
|
+
conn.execute(
|
|
125
|
+
"INSERT INTO project (id, owner_id, name, storage, retention, root,"
|
|
126
|
+
" next_label_idx, created_at, updated_at, sidecar_format)"
|
|
127
|
+
" VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
128
|
+
(
|
|
129
|
+
project.id,
|
|
130
|
+
project.owner_id,
|
|
131
|
+
project.name,
|
|
132
|
+
project.storage,
|
|
133
|
+
project.retention,
|
|
134
|
+
project.root,
|
|
135
|
+
0,
|
|
136
|
+
now,
|
|
137
|
+
now,
|
|
138
|
+
project.sidecar_format,
|
|
139
|
+
),
|
|
140
|
+
)
|
|
141
|
+
return project
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def get(store: Store, project_id: str, owner_id: str = LOCAL_OWNER) -> Project:
|
|
145
|
+
"""Resolve a project *for this owner*, or raise :class:`NotFound`."""
|
|
146
|
+
require_id(project_id, "project id")
|
|
147
|
+
row = store.one(
|
|
148
|
+
"SELECT * FROM project WHERE id = ? AND owner_id = ?", (project_id, owner_id)
|
|
149
|
+
)
|
|
150
|
+
if row is None:
|
|
151
|
+
raise NotFound("no such project")
|
|
152
|
+
return _row_to_project(row)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def reload(store: Store, project: Project) -> Project:
|
|
156
|
+
"""Re-read a project, picking up a moved label counter."""
|
|
157
|
+
return get(store, project.id, project.owner_id)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def listing(store: Store, owner_id: str = LOCAL_OWNER) -> list[Project]:
|
|
161
|
+
rows = store.query(
|
|
162
|
+
"SELECT * FROM project WHERE owner_id = ? ORDER BY updated_at DESC", (owner_id,)
|
|
163
|
+
)
|
|
164
|
+
return [_row_to_project(r) for r in rows]
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def rename(store: Store, project: Project, name: str) -> Project:
|
|
168
|
+
clean = (name or "").strip()
|
|
169
|
+
if not clean:
|
|
170
|
+
raise StoreError("a project needs a name")
|
|
171
|
+
with store.write() as conn:
|
|
172
|
+
conn.execute(
|
|
173
|
+
"UPDATE project SET name = ?, updated_at = ? WHERE id = ? AND owner_id = ?",
|
|
174
|
+
(clean, time.time(), project.id, project.owner_id),
|
|
175
|
+
)
|
|
176
|
+
return reload(store, project)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def delete(store: Store, project: Project) -> None:
|
|
180
|
+
"""Remove a project and everything under it.
|
|
181
|
+
|
|
182
|
+
The cascade is the schema's job (``ON DELETE CASCADE`` plus the
|
|
183
|
+
``foreign_keys`` pragma), not a hand-written sequence of DELETEs that a
|
|
184
|
+
future table would be forgotten from.
|
|
185
|
+
"""
|
|
186
|
+
with store.write() as conn:
|
|
187
|
+
conn.execute(
|
|
188
|
+
"DELETE FROM project WHERE id = ? AND owner_id = ?",
|
|
189
|
+
(project.id, project.owner_id),
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def touch(conn: Any, project_id: str) -> None:
|
|
194
|
+
"""Mark a project as worked on. Called inside an existing transaction.
|
|
195
|
+
|
|
196
|
+
Project mtime is a database column and not the root directory's, because a
|
|
197
|
+
deep write does not bump a parent's mtime -- which is how a project someone
|
|
198
|
+
annotated for a day ends up looking untouched to a sweep.
|
|
199
|
+
"""
|
|
200
|
+
conn.execute("UPDATE project SET updated_at = ? WHERE id = ?", (time.time(), project_id))
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def audit(conn: Any, project: Project, action: str, detail: str = "") -> None:
|
|
204
|
+
conn.execute(
|
|
205
|
+
"INSERT INTO audit (project_id, owner_id, action, detail, at) VALUES (?, ?, ?, ?, ?)",
|
|
206
|
+
(project.id, project.owner_id, action, detail, time.time()),
|
|
207
|
+
)
|