easyannotate 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_factory/__init__.py +82 -0
- data_factory/__main__.py +6 -0
- data_factory/annotate/__init__.py +15 -0
- data_factory/annotate/attach.py +398 -0
- data_factory/annotate/demo.py +161 -0
- data_factory/annotate/images.py +443 -0
- data_factory/annotate/labels.py +399 -0
- data_factory/annotate/loop.py +359 -0
- data_factory/annotate/projects.py +207 -0
- data_factory/annotate/review.py +471 -0
- data_factory/annotate/schema.sql +171 -0
- data_factory/annotate/shapes.py +488 -0
- data_factory/annotate/store.py +292 -0
- data_factory/annotate/tools/__init__.py +39 -0
- data_factory/annotate/tools/fake.py +170 -0
- data_factory/annotate/tools/install.py +254 -0
- data_factory/annotate/tools/local.py +389 -0
- data_factory/annotate/tools/postprocess.py +175 -0
- data_factory/annotate/tools/protocol.py +238 -0
- data_factory/annotate/tools/registry.py +196 -0
- data_factory/annotate/tools/results.py +83 -0
- data_factory/annotate/tools/serve.py +106 -0
- data_factory/cli/__init__.py +21 -0
- data_factory/cli/main.py +837 -0
- data_factory/cli/menu.py +173 -0
- data_factory/cli/render.py +158 -0
- data_factory/core/__init__.py +11 -0
- data_factory/core/annotations/__init__.py +5 -0
- data_factory/core/annotations/coco.py +394 -0
- data_factory/core/annotations/csvfmt.py +174 -0
- data_factory/core/annotations/dota.py +98 -0
- data_factory/core/annotations/formats.py +626 -0
- data_factory/core/annotations/labelme.py +153 -0
- data_factory/core/annotations/model.py +334 -0
- data_factory/core/annotations/voc.py +231 -0
- data_factory/core/annotations/yolo.py +283 -0
- data_factory/core/bktree.py +89 -0
- data_factory/core/fs.py +245 -0
- data_factory/core/imaging.py +626 -0
- data_factory/core/ops/__init__.py +44 -0
- data_factory/core/ops/analyze.py +383 -0
- data_factory/core/ops/annotate_export.py +326 -0
- data_factory/core/ops/audit.py +274 -0
- data_factory/core/ops/augment.py +454 -0
- data_factory/core/ops/collect.py +241 -0
- data_factory/core/ops/combine.py +582 -0
- data_factory/core/ops/convert.py +212 -0
- data_factory/core/ops/dataset.py +246 -0
- data_factory/core/ops/dedup.py +235 -0
- data_factory/core/ops/export.py +168 -0
- data_factory/core/ops/image.py +407 -0
- data_factory/core/ops/labels.py +215 -0
- data_factory/core/ops/pairing.py +133 -0
- data_factory/core/ops/rename.py +154 -0
- data_factory/core/ops/report.py +323 -0
- data_factory/core/ops/visualize.py +164 -0
- data_factory/core/pairs.py +144 -0
- data_factory/core/parallel.py +191 -0
- data_factory/core/provenance.py +136 -0
- data_factory/core/registry.py +339 -0
- data_factory/core/stats.py +403 -0
- data_factory/core/types.py +148 -0
- data_factory/core/writer.py +194 -0
- data_factory/desktop.py +207 -0
- data_factory/hardware.py +243 -0
- data_factory/pipeline.py +192 -0
- data_factory/py.typed +0 -0
- data_factory/service/__init__.py +26 -0
- data_factory/service/app.py +483 -0
- data_factory/service/binding.py +174 -0
- data_factory/service/deps.py +133 -0
- data_factory/service/jobs.py +177 -0
- data_factory/service/limits.py +71 -0
- data_factory/service/routers/__init__.py +11 -0
- data_factory/service/routers/annotate.py +582 -0
- data_factory/service/routers/local.py +464 -0
- data_factory/service/routers/media.py +154 -0
- data_factory/service/routers/predictions.py +116 -0
- data_factory/service/routers/promote.py +182 -0
- data_factory/service/routers/tools.py +308 -0
- data_factory/service/static/annotate/api.js +246 -0
- data_factory/service/static/annotate/app.js +1696 -0
- data_factory/service/static/annotate/assist.js +106 -0
- data_factory/service/static/annotate/filmstrip.js +286 -0
- data_factory/service/static/annotate/hit.js +146 -0
- data_factory/service/static/annotate/labels.js +197 -0
- data_factory/service/static/annotate/render.js +286 -0
- data_factory/service/static/annotate/scene.js +277 -0
- data_factory/service/static/annotate/shapelist.js +72 -0
- data_factory/service/static/annotate/tools/box.js +96 -0
- data_factory/service/static/annotate/tools/line.js +96 -0
- data_factory/service/static/annotate/tools/obb.js +88 -0
- data_factory/service/static/annotate/tools/point.js +49 -0
- data_factory/service/static/annotate/tools/polygon.js +110 -0
- data_factory/service/static/annotate/tools/select.js +227 -0
- data_factory/service/static/annotate/undo.js +176 -0
- data_factory/service/static/annotate/viewport.js +156 -0
- data_factory/service/static/annotate.html +390 -0
- data_factory/service/static/index.html +632 -0
- data_factory/service/workspace.py +244 -0
- easyannotate/__init__.py +64 -0
- easyannotate/__main__.py +5 -0
- easyannotate/project.py +181 -0
- easyannotate/py.typed +0 -0
- easyannotate-0.4.0.dist-info/METADATA +767 -0
- easyannotate-0.4.0.dist-info/RECORD +109 -0
- easyannotate-0.4.0.dist-info/WHEEL +4 -0
- easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
- easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
"""Sandboxed per-dataset storage for the hosted service.
|
|
2
|
+
|
|
3
|
+
The example API shipped with this project takes filesystem paths straight from
|
|
4
|
+
the request body. That is fine on a laptop and catastrophic on a public host:
|
|
5
|
+
``{"folder": "/etc"}`` is an arbitrary-read primitive, and ``{"output": "/"}``
|
|
6
|
+
is worse. Nothing in this module trusts a client-supplied path.
|
|
7
|
+
|
|
8
|
+
Two rules, enforced here rather than remembered at each call site:
|
|
9
|
+
|
|
10
|
+
1. A client never names a directory. It names a **dataset id** -- an opaque
|
|
11
|
+
token this module issued -- and every path is derived from that.
|
|
12
|
+
2. Every derived path is resolved and checked to be inside the workspace root
|
|
13
|
+
before it is used, so symlinks and ``..`` cannot escape.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
import secrets
|
|
21
|
+
import shutil
|
|
22
|
+
import time
|
|
23
|
+
import zipfile
|
|
24
|
+
from collections.abc import Iterator
|
|
25
|
+
from dataclasses import dataclass
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
|
|
28
|
+
#: Ids are opaque and unguessable: possession of the id is the only
|
|
29
|
+
#: authorisation the anonymous flow has, so it must not be enumerable.
|
|
30
|
+
ID_BYTES = 16
|
|
31
|
+
ID_PATTERN = re.compile(r"^[0-9a-f]{32}$")
|
|
32
|
+
|
|
33
|
+
#: Upload limits. A zip bomb is a small file that expands to fill the disk, so
|
|
34
|
+
#: the compressed size alone is not a sufficient guard.
|
|
35
|
+
MAX_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024 # 2 GB compressed
|
|
36
|
+
MAX_EXTRACTED_BYTES = 8 * 1024 * 1024 * 1024 # 8 GB uncompressed
|
|
37
|
+
MAX_MEMBERS = 200_000
|
|
38
|
+
MAX_RATIO = 200 # uncompressed / compressed
|
|
39
|
+
|
|
40
|
+
#: Anonymous datasets are transient. Anything older than this is swept.
|
|
41
|
+
DEFAULT_TTL_SECONDS = 24 * 60 * 60
|
|
42
|
+
|
|
43
|
+
SAFE_NAME = re.compile(r"[^A-Za-z0-9._-]")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class WorkspaceError(Exception):
|
|
47
|
+
"""Raised for anything a client did wrong with storage."""
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True)
|
|
51
|
+
class Dataset:
|
|
52
|
+
"""One sandboxed dataset directory."""
|
|
53
|
+
|
|
54
|
+
id: str
|
|
55
|
+
root: Path
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def source(self) -> Path:
|
|
59
|
+
"""Where the uploaded dataset lives. Operations read from here."""
|
|
60
|
+
return self.root / "source"
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def outputs(self) -> Path:
|
|
64
|
+
"""Parent of every generated dataset. Operations write under here."""
|
|
65
|
+
return self.root / "outputs"
|
|
66
|
+
|
|
67
|
+
def output(self, name: str) -> Path:
|
|
68
|
+
"""A named output directory, with the name sanitised to one segment."""
|
|
69
|
+
return self.outputs / safe_segment(name)
|
|
70
|
+
|
|
71
|
+
def created_at(self) -> float:
|
|
72
|
+
try:
|
|
73
|
+
return self.root.stat().st_mtime
|
|
74
|
+
except OSError:
|
|
75
|
+
return 0.0
|
|
76
|
+
|
|
77
|
+
def size_bytes(self) -> int:
|
|
78
|
+
return sum(p.stat().st_size for p in self.root.rglob("*") if p.is_file())
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class Workspace:
|
|
82
|
+
"""The root under which every dataset sandbox lives."""
|
|
83
|
+
|
|
84
|
+
def __init__(self, root: Path, *, ttl_seconds: int = DEFAULT_TTL_SECONDS) -> None:
|
|
85
|
+
self.root = Path(root).resolve()
|
|
86
|
+
self.root.mkdir(parents=True, exist_ok=True)
|
|
87
|
+
self.ttl_seconds = ttl_seconds
|
|
88
|
+
|
|
89
|
+
# -- lifecycle ---------------------------------------------------------
|
|
90
|
+
def create(self) -> Dataset:
|
|
91
|
+
dataset_id = secrets.token_hex(ID_BYTES)
|
|
92
|
+
dataset = Dataset(dataset_id, self.root / dataset_id)
|
|
93
|
+
dataset.source.mkdir(parents=True, exist_ok=True)
|
|
94
|
+
dataset.outputs.mkdir(parents=True, exist_ok=True)
|
|
95
|
+
return dataset
|
|
96
|
+
|
|
97
|
+
def get(self, dataset_id: str) -> Dataset:
|
|
98
|
+
"""Resolve an id to a sandbox, rejecting anything that is not one.
|
|
99
|
+
|
|
100
|
+
The pattern check matters more than it looks: without it, an id of
|
|
101
|
+
``..`` would resolve to the workspace parent.
|
|
102
|
+
"""
|
|
103
|
+
if not ID_PATTERN.match(dataset_id or ""):
|
|
104
|
+
raise WorkspaceError("not a valid dataset id")
|
|
105
|
+
root = (self.root / dataset_id).resolve()
|
|
106
|
+
self.assert_inside(root)
|
|
107
|
+
if not root.is_dir():
|
|
108
|
+
raise WorkspaceError("unknown dataset -- it may have expired")
|
|
109
|
+
return Dataset(dataset_id, root)
|
|
110
|
+
|
|
111
|
+
def delete(self, dataset_id: str) -> None:
|
|
112
|
+
dataset = self.get(dataset_id)
|
|
113
|
+
shutil.rmtree(dataset.root, ignore_errors=True)
|
|
114
|
+
|
|
115
|
+
def sweep(self) -> int:
|
|
116
|
+
"""Remove datasets past their TTL. Returns how many were removed."""
|
|
117
|
+
cutoff = time.time() - self.ttl_seconds
|
|
118
|
+
removed = 0
|
|
119
|
+
for child in self.root.iterdir():
|
|
120
|
+
if not child.is_dir() or not ID_PATTERN.match(child.name):
|
|
121
|
+
continue
|
|
122
|
+
try:
|
|
123
|
+
if child.stat().st_mtime < cutoff:
|
|
124
|
+
shutil.rmtree(child, ignore_errors=True)
|
|
125
|
+
removed += 1
|
|
126
|
+
except OSError:
|
|
127
|
+
continue
|
|
128
|
+
return removed
|
|
129
|
+
|
|
130
|
+
def datasets(self) -> Iterator[Dataset]:
|
|
131
|
+
for child in sorted(self.root.iterdir()):
|
|
132
|
+
if child.is_dir() and ID_PATTERN.match(child.name):
|
|
133
|
+
yield Dataset(child.name, child)
|
|
134
|
+
|
|
135
|
+
# -- the security boundary --------------------------------------------
|
|
136
|
+
def assert_inside(self, path: Path) -> Path:
|
|
137
|
+
"""Fail unless ``path`` really is under the workspace root.
|
|
138
|
+
|
|
139
|
+
Uses the resolved path, so a symlink planted inside a sandbox cannot
|
|
140
|
+
redirect a write to somewhere else on the host.
|
|
141
|
+
"""
|
|
142
|
+
resolved = Path(path).resolve()
|
|
143
|
+
if resolved != self.root and self.root not in resolved.parents:
|
|
144
|
+
raise WorkspaceError("path escapes the workspace")
|
|
145
|
+
return resolved
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def safe_segment(name: str) -> str:
|
|
149
|
+
"""Reduce arbitrary text to one filesystem-safe path segment."""
|
|
150
|
+
cleaned = SAFE_NAME.sub("_", (name or "").strip())[:64].strip("._")
|
|
151
|
+
return cleaned or "output"
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
# ------------------------------------------------------------------ uploads
|
|
155
|
+
|
|
156
|
+
def extract_zip(archive: Path, destination: Path, *, workspace: Workspace) -> dict[str, int]:
|
|
157
|
+
"""Unpack an uploaded zip safely.
|
|
158
|
+
|
|
159
|
+
Guards, in the order they matter:
|
|
160
|
+
|
|
161
|
+
* **Zip slip** -- a member named ``../../etc/cron.d/x`` writes outside the
|
|
162
|
+
target. Every member is resolved and checked against the destination.
|
|
163
|
+
* **Zip bomb** -- a few kB expanding to fill the disk. Both the total
|
|
164
|
+
uncompressed size and the per-member compression ratio are capped.
|
|
165
|
+
* **Member count** -- a million tiny files is its own denial of service.
|
|
166
|
+
"""
|
|
167
|
+
destination.mkdir(parents=True, exist_ok=True)
|
|
168
|
+
resolved_destination = workspace.assert_inside(destination)
|
|
169
|
+
|
|
170
|
+
written = 0
|
|
171
|
+
total = 0
|
|
172
|
+
skipped = 0
|
|
173
|
+
|
|
174
|
+
try:
|
|
175
|
+
with zipfile.ZipFile(archive) as bundle:
|
|
176
|
+
members = bundle.infolist()
|
|
177
|
+
if len(members) > MAX_MEMBERS:
|
|
178
|
+
raise WorkspaceError(f"archive holds more than {MAX_MEMBERS:,} entries")
|
|
179
|
+
|
|
180
|
+
declared = sum(m.file_size for m in members)
|
|
181
|
+
if declared > MAX_EXTRACTED_BYTES:
|
|
182
|
+
raise WorkspaceError(
|
|
183
|
+
f"archive expands to {declared / 1e9:.1f} GB, over the "
|
|
184
|
+
f"{MAX_EXTRACTED_BYTES / 1e9:.0f} GB limit"
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
for member in members:
|
|
188
|
+
if member.is_dir():
|
|
189
|
+
continue
|
|
190
|
+
name = member.filename.replace("\\", "/")
|
|
191
|
+
if name.startswith("/") or ".." in Path(name).parts:
|
|
192
|
+
skipped += 1
|
|
193
|
+
continue
|
|
194
|
+
# Some archives carry junk; keeping it would confuse format
|
|
195
|
+
# detection more than dropping it loses anything.
|
|
196
|
+
if Path(name).name.startswith("._") or "__MACOSX/" in name:
|
|
197
|
+
skipped += 1
|
|
198
|
+
continue
|
|
199
|
+
|
|
200
|
+
target = (resolved_destination / name).resolve()
|
|
201
|
+
if resolved_destination not in target.parents and target != resolved_destination:
|
|
202
|
+
skipped += 1
|
|
203
|
+
continue
|
|
204
|
+
|
|
205
|
+
if member.compress_size and member.file_size / max(1, member.compress_size) > MAX_RATIO:
|
|
206
|
+
raise WorkspaceError(f"{name}: suspicious compression ratio, refusing to extract")
|
|
207
|
+
|
|
208
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
209
|
+
with bundle.open(member) as src, target.open("wb") as dst:
|
|
210
|
+
# Copy in chunks and stop at the cap, rather than trusting
|
|
211
|
+
# the header's declared size, which an attacker controls.
|
|
212
|
+
while chunk := src.read(1 << 20):
|
|
213
|
+
total += len(chunk)
|
|
214
|
+
if total > MAX_EXTRACTED_BYTES:
|
|
215
|
+
raise WorkspaceError("archive exceeded the uncompressed size limit")
|
|
216
|
+
dst.write(chunk)
|
|
217
|
+
written += 1
|
|
218
|
+
except zipfile.BadZipFile as exc:
|
|
219
|
+
raise WorkspaceError(f"not a readable zip archive ({exc})") from exc
|
|
220
|
+
|
|
221
|
+
if written == 0:
|
|
222
|
+
raise WorkspaceError("the archive contained no usable files")
|
|
223
|
+
return {"files": written, "bytes": total, "skipped": skipped}
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def make_zip(source: Path, archive: Path, *, workspace: Workspace) -> Path:
|
|
227
|
+
"""Bundle a result directory for download."""
|
|
228
|
+
workspace.assert_inside(source)
|
|
229
|
+
if not source.is_dir():
|
|
230
|
+
raise WorkspaceError("nothing to download yet")
|
|
231
|
+
archive.parent.mkdir(parents=True, exist_ok=True)
|
|
232
|
+
with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as bundle:
|
|
233
|
+
for path in sorted(source.rglob("*")):
|
|
234
|
+
if path.is_file():
|
|
235
|
+
bundle.write(path, path.relative_to(source).as_posix())
|
|
236
|
+
return archive
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def resolve_workspace_root() -> Path:
|
|
240
|
+
"""Where datasets live, overridable for containers and tests."""
|
|
241
|
+
configured = os.environ.get("DATAFACTORY_WORKSPACE")
|
|
242
|
+
if configured:
|
|
243
|
+
return Path(configured)
|
|
244
|
+
return Path.home() / ".datafactory" / "workspaces"
|
easyannotate/__init__.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""EasyAnnotate as a library: headless, scriptable, no server, no browser.
|
|
2
|
+
|
|
3
|
+
import easyannotate as ea
|
|
4
|
+
|
|
5
|
+
project = ea.open_folder("./doors") # labels already there are loaded
|
|
6
|
+
print(project.counts()) # {'todo': 812, 'reviewed': 200, ...}
|
|
7
|
+
project.import_predictions("runs/predict/labels")
|
|
8
|
+
project.export("./train-set", to="yolo") # reviewed images only, stable class indices
|
|
9
|
+
|
|
10
|
+
ea.run("inspect.health", {"folder": "./doors"}) # any of the dataset operations
|
|
11
|
+
|
|
12
|
+
The same database the editor and the CLI use, so a script, a terminal and a
|
|
13
|
+
browser can take turns on one project. Training and prediction need the
|
|
14
|
+
model tools (``easyannotate tools install``); everything else needs nothing.
|
|
15
|
+
|
|
16
|
+
The implementation lives in ``data_factory``, the package's name before the
|
|
17
|
+
editor became the point; this module is the stable surface over it.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from data_factory import (
|
|
23
|
+
Context,
|
|
24
|
+
OpResult,
|
|
25
|
+
ParamError,
|
|
26
|
+
__version__,
|
|
27
|
+
get_operation,
|
|
28
|
+
list_operations,
|
|
29
|
+
run,
|
|
30
|
+
schema,
|
|
31
|
+
)
|
|
32
|
+
from data_factory.annotate.attach import AttachError
|
|
33
|
+
from data_factory.annotate.loop import LoopError
|
|
34
|
+
from data_factory.annotate.review import ReviewError
|
|
35
|
+
from data_factory.annotate.store import NotFound, StoreError
|
|
36
|
+
|
|
37
|
+
from .project import Project, close, database_path, list_projects, open_folder, open_project
|
|
38
|
+
|
|
39
|
+
#: Errors about projects, images, labels, predictions and the loop derive from
|
|
40
|
+
#: this. A bad parameter to an operation or export raises ParamError instead.
|
|
41
|
+
Error = StoreError
|
|
42
|
+
|
|
43
|
+
__all__ = [
|
|
44
|
+
"AttachError",
|
|
45
|
+
"Context",
|
|
46
|
+
"Error",
|
|
47
|
+
"LoopError",
|
|
48
|
+
"NotFound",
|
|
49
|
+
"OpResult",
|
|
50
|
+
"ParamError",
|
|
51
|
+
"Project",
|
|
52
|
+
"ReviewError",
|
|
53
|
+
"StoreError",
|
|
54
|
+
"close",
|
|
55
|
+
"__version__",
|
|
56
|
+
"database_path",
|
|
57
|
+
"get_operation",
|
|
58
|
+
"list_operations",
|
|
59
|
+
"list_projects",
|
|
60
|
+
"open_folder",
|
|
61
|
+
"open_project",
|
|
62
|
+
"run",
|
|
63
|
+
"schema",
|
|
64
|
+
]
|
easyannotate/__main__.py
ADDED
easyannotate/project.py
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""A project, as a script sees it."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import threading
|
|
6
|
+
from collections.abc import Callable, Iterable
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from data_factory.annotate import attach, images, labels, loop, projects, review, shapes
|
|
11
|
+
from data_factory.annotate.store import LOCAL_OWNER, Store
|
|
12
|
+
from data_factory.core import registry
|
|
13
|
+
from data_factory.core.types import Context, OpResult
|
|
14
|
+
from data_factory.service.workspace import resolve_workspace_root
|
|
15
|
+
|
|
16
|
+
_stores: dict[Path, Store] = {}
|
|
17
|
+
# Opening a store migrates it; two threads doing that at once raced to
|
|
18
|
+
# "database is locked". One lock, as the server's cache has.
|
|
19
|
+
_stores_lock = threading.Lock()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def database_path() -> Path:
|
|
23
|
+
"""The annotation database the editor, CLI and library share.
|
|
24
|
+
|
|
25
|
+
Beside the workspace -- ``DATAFACTORY_WORKSPACE``'s parent, by default
|
|
26
|
+
``~/.datafactory`` -- exactly where ``easyannotate serve`` keeps it.
|
|
27
|
+
"""
|
|
28
|
+
return resolve_workspace_root().parent / "annotations.db"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _store() -> Store:
|
|
32
|
+
path = database_path().resolve()
|
|
33
|
+
with _stores_lock:
|
|
34
|
+
if path not in _stores:
|
|
35
|
+
_stores[path] = Store(path)
|
|
36
|
+
return _stores[path]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def close() -> None:
|
|
40
|
+
"""Release the database files. Needed before deleting the folder they are
|
|
41
|
+
in -- Windows will not remove an open file."""
|
|
42
|
+
with _stores_lock:
|
|
43
|
+
for store in _stores.values():
|
|
44
|
+
store.close()
|
|
45
|
+
_stores.clear()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def open_folder(folder: str | Path, *, name: str = "", import_annotations: bool = True) -> Project:
|
|
49
|
+
"""Open a folder of images, editing its label files in place.
|
|
50
|
+
|
|
51
|
+
Reopening the same folder returns the same project, with its review state.
|
|
52
|
+
Label files already there (VOC, YOLO, COCO, LabelMe, DOTA) are loaded the
|
|
53
|
+
first time.
|
|
54
|
+
"""
|
|
55
|
+
store = _store()
|
|
56
|
+
summary = attach.open_folder(
|
|
57
|
+
store, Path(folder).expanduser(), name=name, import_annotations=import_annotations
|
|
58
|
+
)
|
|
59
|
+
return Project(store, summary["project"], notes=summary.get("warnings") or [])
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def open_project(project_id: str) -> Project:
|
|
63
|
+
"""A project by id -- one made by uploading in the web interface, say."""
|
|
64
|
+
store = _store()
|
|
65
|
+
return Project(store, projects.get(store, project_id, LOCAL_OWNER))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def list_projects() -> list[Project]:
|
|
69
|
+
store = _store()
|
|
70
|
+
return [Project(store, p) for p in projects.listing(store, LOCAL_OWNER)]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class Project:
|
|
74
|
+
"""One annotation project. Cheap to hold; every call reads the database."""
|
|
75
|
+
|
|
76
|
+
def __init__(self, store: Store, record: projects.Project, *, notes: list[str] | None = None) -> None:
|
|
77
|
+
self._store = store
|
|
78
|
+
self._record = record
|
|
79
|
+
self.notes = list(notes or [])
|
|
80
|
+
|
|
81
|
+
# -- identity ----------------------------------------------------------
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def id(self) -> str:
|
|
85
|
+
return self._record.id
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def name(self) -> str:
|
|
89
|
+
return self._record.name
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def root(self) -> Path:
|
|
93
|
+
"""Where the images are: the folder you opened, or the managed copy."""
|
|
94
|
+
if self._record.is_attached and self._record.root:
|
|
95
|
+
return Path(self._record.root)
|
|
96
|
+
# Beside *this project's* database -- not wherever the environment
|
|
97
|
+
# points now, which may be another workspace by the time this is read.
|
|
98
|
+
return self._store.path.parent / "projects" / self._record.id
|
|
99
|
+
|
|
100
|
+
def __repr__(self) -> str:
|
|
101
|
+
return f"<Project {self.name!r} {self.id[:8]} {self.counts()}>"
|
|
102
|
+
|
|
103
|
+
# -- reading -----------------------------------------------------------
|
|
104
|
+
|
|
105
|
+
def counts(self) -> dict[str, int]:
|
|
106
|
+
"""Images per state: todo, predicted, in_progress, reviewed, skipped, total."""
|
|
107
|
+
return images.counts(self._store, self._record)
|
|
108
|
+
|
|
109
|
+
def classes(self) -> list[str]:
|
|
110
|
+
"""The class list in index order -- what YOLO index *n* means."""
|
|
111
|
+
return labels.class_list(self._store, self._record)
|
|
112
|
+
|
|
113
|
+
def images(self, *, state: str = "", order: str = "", limit: int | None = None,
|
|
114
|
+
offset: int = 0) -> list[dict[str, Any]]:
|
|
115
|
+
"""Images as dicts -- all of them unless ``limit`` says otherwise.
|
|
116
|
+
``order="uncertain"`` puts likely mistakes first."""
|
|
117
|
+
found: list[dict[str, Any]] = []
|
|
118
|
+
page = 5000 # the store's largest page
|
|
119
|
+
while limit is None or len(found) < limit:
|
|
120
|
+
want = page if limit is None else min(page, limit - len(found))
|
|
121
|
+
batch = images.listing(self._store, self._record, state=state, order=order,
|
|
122
|
+
limit=want, offset=offset + len(found))
|
|
123
|
+
found.extend(image.as_dict() for image in batch)
|
|
124
|
+
if len(batch) < want:
|
|
125
|
+
break
|
|
126
|
+
return found
|
|
127
|
+
|
|
128
|
+
def shapes(self, image_id: str) -> list[dict[str, Any]]:
|
|
129
|
+
image = images.get(self._store, self._record, image_id)
|
|
130
|
+
return [s.as_dict() for s in shapes.listing(self._store, self._record, image)]
|
|
131
|
+
|
|
132
|
+
def agreement(self) -> dict[str, Any]:
|
|
133
|
+
"""How often the model was right on reviewed images."""
|
|
134
|
+
return shapes.agreement(self._store, self._record)
|
|
135
|
+
|
|
136
|
+
# -- the loop ----------------------------------------------------------
|
|
137
|
+
|
|
138
|
+
def import_predictions(self, source: str | Path, *, min_confidence: float = 0.0,
|
|
139
|
+
overwrite_reviewed: bool = False) -> dict[str, Any]:
|
|
140
|
+
"""Load a model run (YOLO .txt folder, COCO results .json, VOC, LabelMe)
|
|
141
|
+
as predictions to review. Returns the matching report -- read it."""
|
|
142
|
+
return review.import_predictions(
|
|
143
|
+
self._store, self._record, Path(source).expanduser(),
|
|
144
|
+
min_confidence=min_confidence, overwrite_reviewed=overwrite_reviewed,
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
def predict(self, model: str | Path, *, conf: float = 0.25,
|
|
148
|
+
on_progress: Callable[[int, int, str], None] | None = None) -> dict[str, Any]:
|
|
149
|
+
"""Run an Ultralytics model over the unreviewed images. Needs the tools."""
|
|
150
|
+
return loop.predict(self._store, self._record, self.root, Path(model).expanduser(), conf=conf,
|
|
151
|
+
on_progress=on_progress)
|
|
152
|
+
|
|
153
|
+
def train(self, *, epochs: int = 30, imgsz: int = 640, base: str = "yolov8n.pt",
|
|
154
|
+
on_line: Callable[[str], None] = print) -> dict[str, Any]:
|
|
155
|
+
"""Train on the reviewed images. Returns ``{"model": Path, "metrics": ...}``."""
|
|
156
|
+
return loop.train(self._store, self._record, epochs=epochs, imgsz=imgsz, base=base,
|
|
157
|
+
on_line=on_line)
|
|
158
|
+
|
|
159
|
+
def export(self, output: str | Path, *, to: str = "yolo",
|
|
160
|
+
states: Iterable[str] = ("reviewed",), copy_images: bool = True,
|
|
161
|
+
segmentation: bool = True, dry_run: bool = False) -> OpResult:
|
|
162
|
+
"""Write the project as a dataset. Reviewed images only, by default --
|
|
163
|
+
exporting unreviewed model output into training is how a model poisons
|
|
164
|
+
itself. Class indices are the project's own and never move."""
|
|
165
|
+
if isinstance(states, str):
|
|
166
|
+
states = [states] # "reviewed", not r, e, v, i, e, w, e, d
|
|
167
|
+
# Through Operation.run, like every other caller: the same validation
|
|
168
|
+
# (a bad format is a ParamError), `~` expanded, and provenance written.
|
|
169
|
+
return registry.get("annotate.export").run(
|
|
170
|
+
{
|
|
171
|
+
"database": str(self._store.path),
|
|
172
|
+
"project": self._record.id,
|
|
173
|
+
"output": str(output),
|
|
174
|
+
"to": to,
|
|
175
|
+
"states": list(states),
|
|
176
|
+
"owner": self._record.owner_id,
|
|
177
|
+
"copy_images": copy_images,
|
|
178
|
+
"segmentation": segmentation,
|
|
179
|
+
},
|
|
180
|
+
Context(dry_run=dry_run),
|
|
181
|
+
)
|
easyannotate/py.typed
ADDED
|
File without changes
|