easyannotate 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_factory/__init__.py +82 -0
- data_factory/__main__.py +6 -0
- data_factory/annotate/__init__.py +15 -0
- data_factory/annotate/attach.py +398 -0
- data_factory/annotate/demo.py +161 -0
- data_factory/annotate/images.py +443 -0
- data_factory/annotate/labels.py +399 -0
- data_factory/annotate/loop.py +359 -0
- data_factory/annotate/projects.py +207 -0
- data_factory/annotate/review.py +471 -0
- data_factory/annotate/schema.sql +171 -0
- data_factory/annotate/shapes.py +488 -0
- data_factory/annotate/store.py +292 -0
- data_factory/annotate/tools/__init__.py +39 -0
- data_factory/annotate/tools/fake.py +170 -0
- data_factory/annotate/tools/install.py +254 -0
- data_factory/annotate/tools/local.py +389 -0
- data_factory/annotate/tools/postprocess.py +175 -0
- data_factory/annotate/tools/protocol.py +238 -0
- data_factory/annotate/tools/registry.py +196 -0
- data_factory/annotate/tools/results.py +83 -0
- data_factory/annotate/tools/serve.py +106 -0
- data_factory/cli/__init__.py +21 -0
- data_factory/cli/main.py +837 -0
- data_factory/cli/menu.py +173 -0
- data_factory/cli/render.py +158 -0
- data_factory/core/__init__.py +11 -0
- data_factory/core/annotations/__init__.py +5 -0
- data_factory/core/annotations/coco.py +394 -0
- data_factory/core/annotations/csvfmt.py +174 -0
- data_factory/core/annotations/dota.py +98 -0
- data_factory/core/annotations/formats.py +626 -0
- data_factory/core/annotations/labelme.py +153 -0
- data_factory/core/annotations/model.py +334 -0
- data_factory/core/annotations/voc.py +231 -0
- data_factory/core/annotations/yolo.py +283 -0
- data_factory/core/bktree.py +89 -0
- data_factory/core/fs.py +245 -0
- data_factory/core/imaging.py +626 -0
- data_factory/core/ops/__init__.py +44 -0
- data_factory/core/ops/analyze.py +383 -0
- data_factory/core/ops/annotate_export.py +326 -0
- data_factory/core/ops/audit.py +274 -0
- data_factory/core/ops/augment.py +454 -0
- data_factory/core/ops/collect.py +241 -0
- data_factory/core/ops/combine.py +582 -0
- data_factory/core/ops/convert.py +212 -0
- data_factory/core/ops/dataset.py +246 -0
- data_factory/core/ops/dedup.py +235 -0
- data_factory/core/ops/export.py +168 -0
- data_factory/core/ops/image.py +407 -0
- data_factory/core/ops/labels.py +215 -0
- data_factory/core/ops/pairing.py +133 -0
- data_factory/core/ops/rename.py +154 -0
- data_factory/core/ops/report.py +323 -0
- data_factory/core/ops/visualize.py +164 -0
- data_factory/core/pairs.py +144 -0
- data_factory/core/parallel.py +191 -0
- data_factory/core/provenance.py +136 -0
- data_factory/core/registry.py +339 -0
- data_factory/core/stats.py +403 -0
- data_factory/core/types.py +148 -0
- data_factory/core/writer.py +194 -0
- data_factory/desktop.py +207 -0
- data_factory/hardware.py +243 -0
- data_factory/pipeline.py +192 -0
- data_factory/py.typed +0 -0
- data_factory/service/__init__.py +26 -0
- data_factory/service/app.py +483 -0
- data_factory/service/binding.py +174 -0
- data_factory/service/deps.py +133 -0
- data_factory/service/jobs.py +177 -0
- data_factory/service/limits.py +71 -0
- data_factory/service/routers/__init__.py +11 -0
- data_factory/service/routers/annotate.py +582 -0
- data_factory/service/routers/local.py +464 -0
- data_factory/service/routers/media.py +154 -0
- data_factory/service/routers/predictions.py +116 -0
- data_factory/service/routers/promote.py +182 -0
- data_factory/service/routers/tools.py +308 -0
- data_factory/service/static/annotate/api.js +246 -0
- data_factory/service/static/annotate/app.js +1696 -0
- data_factory/service/static/annotate/assist.js +106 -0
- data_factory/service/static/annotate/filmstrip.js +286 -0
- data_factory/service/static/annotate/hit.js +146 -0
- data_factory/service/static/annotate/labels.js +197 -0
- data_factory/service/static/annotate/render.js +286 -0
- data_factory/service/static/annotate/scene.js +277 -0
- data_factory/service/static/annotate/shapelist.js +72 -0
- data_factory/service/static/annotate/tools/box.js +96 -0
- data_factory/service/static/annotate/tools/line.js +96 -0
- data_factory/service/static/annotate/tools/obb.js +88 -0
- data_factory/service/static/annotate/tools/point.js +49 -0
- data_factory/service/static/annotate/tools/polygon.js +110 -0
- data_factory/service/static/annotate/tools/select.js +227 -0
- data_factory/service/static/annotate/undo.js +176 -0
- data_factory/service/static/annotate/viewport.js +156 -0
- data_factory/service/static/annotate.html +390 -0
- data_factory/service/static/index.html +632 -0
- data_factory/service/workspace.py +244 -0
- easyannotate/__init__.py +64 -0
- easyannotate/__main__.py +5 -0
- easyannotate/project.py +181 -0
- easyannotate/py.typed +0 -0
- easyannotate-0.4.0.dist-info/METADATA +767 -0
- easyannotate-0.4.0.dist-info/RECORD +109 -0
- easyannotate-0.4.0.dist-info/WHEEL +4 -0
- easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
- easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
data_factory/pipeline.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""Run a recipe: an ordered list of operations described in YAML or JSON.
|
|
2
|
+
|
|
3
|
+
A dataset build is rarely one command. Writing the sequence down makes it
|
|
4
|
+
reproducible and reviewable, and means the same build can be re-run later, by
|
|
5
|
+
someone else, or from a web backend, without anyone remembering the order.
|
|
6
|
+
|
|
7
|
+
name: door-v4
|
|
8
|
+
vars:
|
|
9
|
+
root: D:/Data_factory/Projects/Project_Door/working
|
|
10
|
+
steps:
|
|
11
|
+
- op: clean.orphans
|
|
12
|
+
with: {folder: "${root}", scope: labels, action: delete}
|
|
13
|
+
- op: labels.rename
|
|
14
|
+
with: {folder: "${root}", fold_case: true}
|
|
15
|
+
- op: dataset.split
|
|
16
|
+
with: {folder: "${root}", output: "${root}/../splits"}
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
from dataclasses import dataclass, field
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from .core import registry
|
|
27
|
+
from .core.types import Context, OpResult
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class RecipeError(ValueError):
|
|
31
|
+
"""Raised when a recipe file is malformed or names an unknown operation."""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class Step:
|
|
36
|
+
op: str
|
|
37
|
+
params: dict[str, Any] = field(default_factory=dict)
|
|
38
|
+
name: str = ""
|
|
39
|
+
skip: bool = False
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def label(self) -> str:
|
|
43
|
+
return self.name or self.op
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class Recipe:
|
|
48
|
+
name: str
|
|
49
|
+
steps: list[Step]
|
|
50
|
+
description: str = ""
|
|
51
|
+
variables: dict[str, str] = field(default_factory=dict)
|
|
52
|
+
|
|
53
|
+
@classmethod
|
|
54
|
+
def load(cls, path: Path | str, overrides: dict[str, str] | None = None) -> Recipe:
|
|
55
|
+
# Accept a plain string too: the CLI always hands over a Path, but a
|
|
56
|
+
# library caller reasonably writes Recipe.load("build.yaml").
|
|
57
|
+
path = Path(path)
|
|
58
|
+
document = _load_document(path)
|
|
59
|
+
return cls.from_dict(document, source=path, overrides=overrides)
|
|
60
|
+
|
|
61
|
+
@classmethod
|
|
62
|
+
def from_dict(
|
|
63
|
+
cls,
|
|
64
|
+
document: dict[str, Any],
|
|
65
|
+
source: Path | None = None,
|
|
66
|
+
overrides: dict[str, str] | None = None,
|
|
67
|
+
) -> Recipe:
|
|
68
|
+
if not isinstance(document, dict):
|
|
69
|
+
raise RecipeError("a recipe must be a mapping at the top level")
|
|
70
|
+
raw_steps = document.get("steps")
|
|
71
|
+
if not isinstance(raw_steps, list) or not raw_steps:
|
|
72
|
+
raise RecipeError("a recipe needs a non-empty 'steps' list")
|
|
73
|
+
|
|
74
|
+
# Overrides win over the file's own vars, so one recipe can be pointed
|
|
75
|
+
# at a different dataset without being edited.
|
|
76
|
+
variables = {str(k): str(v) for k, v in (document.get("vars") or {}).items()}
|
|
77
|
+
variables.update({str(k): str(v) for k, v in (overrides or {}).items()})
|
|
78
|
+
steps: list[Step] = []
|
|
79
|
+
for position, entry in enumerate(raw_steps, start=1):
|
|
80
|
+
if not isinstance(entry, dict) or "op" not in entry:
|
|
81
|
+
raise RecipeError(f"step {position}: each step needs an 'op' key")
|
|
82
|
+
op_name = str(entry["op"])
|
|
83
|
+
try:
|
|
84
|
+
operation = registry.get(op_name)
|
|
85
|
+
except KeyError as exc:
|
|
86
|
+
raise RecipeError(f"step {position}: {exc}") from None
|
|
87
|
+
|
|
88
|
+
params = entry.get("with") or entry.get("params") or {}
|
|
89
|
+
if not isinstance(params, dict):
|
|
90
|
+
raise RecipeError(f"step {position} ({op_name}): 'with' must be a mapping")
|
|
91
|
+
params = {k: _substitute(v, variables) for k, v in params.items()}
|
|
92
|
+
|
|
93
|
+
# Validate now rather than three steps into a long run -- but do not
|
|
94
|
+
# check that paths exist, since a step's input is often the previous
|
|
95
|
+
# step's output and will not be there until the run reaches it.
|
|
96
|
+
try:
|
|
97
|
+
operation.bind(params, check_paths=False)
|
|
98
|
+
except registry.ParamError as exc:
|
|
99
|
+
raise RecipeError(f"step {position} ({op_name}): {exc}") from None
|
|
100
|
+
|
|
101
|
+
steps.append(
|
|
102
|
+
Step(op=op_name, params=params, name=str(entry.get("name", "")),
|
|
103
|
+
skip=bool(entry.get("skip", False)))
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
return cls(
|
|
107
|
+
name=str(document.get("name") or (source.stem if source else "recipe")),
|
|
108
|
+
description=str(document.get("description", "")),
|
|
109
|
+
variables=variables,
|
|
110
|
+
steps=steps,
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _substitute(value: Any, variables: dict[str, str]) -> Any:
|
|
115
|
+
"""Expand ``${name}`` references, recursing through lists and mappings."""
|
|
116
|
+
if isinstance(value, str):
|
|
117
|
+
for key, replacement in variables.items():
|
|
118
|
+
value = value.replace("${" + key + "}", replacement)
|
|
119
|
+
return value
|
|
120
|
+
if isinstance(value, list):
|
|
121
|
+
return [_substitute(v, variables) for v in value]
|
|
122
|
+
if isinstance(value, dict):
|
|
123
|
+
return {k: _substitute(v, variables) for k, v in value.items()}
|
|
124
|
+
return value
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _load_document(path: Path) -> dict[str, Any]:
|
|
128
|
+
text = path.read_text(encoding="utf-8")
|
|
129
|
+
if path.suffix.lower() in (".yaml", ".yml"):
|
|
130
|
+
try:
|
|
131
|
+
import yaml
|
|
132
|
+
except ImportError as exc:
|
|
133
|
+
raise RecipeError(
|
|
134
|
+
f"reading {path.name} needs PyYAML installed, or write the recipe as .json ({exc})"
|
|
135
|
+
) from None
|
|
136
|
+
return yaml.safe_load(text) or {}
|
|
137
|
+
return json.loads(text)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
@dataclass
|
|
141
|
+
class RunReport:
|
|
142
|
+
"""Outcome of a whole recipe, one entry per executed step."""
|
|
143
|
+
|
|
144
|
+
recipe: str
|
|
145
|
+
results: list[OpResult] = field(default_factory=list)
|
|
146
|
+
stopped_at: str | None = None
|
|
147
|
+
|
|
148
|
+
@property
|
|
149
|
+
def ok(self) -> bool:
|
|
150
|
+
return self.stopped_at is None and all(r.ok for r in self.results)
|
|
151
|
+
|
|
152
|
+
def as_dict(self) -> dict[str, Any]:
|
|
153
|
+
return {
|
|
154
|
+
"recipe": self.recipe,
|
|
155
|
+
"ok": self.ok,
|
|
156
|
+
"stopped_at": self.stopped_at,
|
|
157
|
+
"steps": [r.as_dict(include_actions=False) for r in self.results],
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def run(
|
|
162
|
+
recipe: Recipe,
|
|
163
|
+
ctx: Context | None = None,
|
|
164
|
+
*,
|
|
165
|
+
continue_on_error: bool = False,
|
|
166
|
+
on_step: Any = None,
|
|
167
|
+
) -> RunReport:
|
|
168
|
+
"""Execute every step in order.
|
|
169
|
+
|
|
170
|
+
Stops at the first failure by default: later steps usually assume the
|
|
171
|
+
earlier ones succeeded, and carrying on with a half-cleaned folder does more
|
|
172
|
+
damage than stopping.
|
|
173
|
+
"""
|
|
174
|
+
ctx = ctx or Context()
|
|
175
|
+
report = RunReport(recipe=recipe.name)
|
|
176
|
+
|
|
177
|
+
for position, step in enumerate(recipe.steps, start=1):
|
|
178
|
+
if step.skip:
|
|
179
|
+
continue
|
|
180
|
+
ctx.check_cancelled()
|
|
181
|
+
if on_step is not None:
|
|
182
|
+
on_step(position, len(recipe.steps), step)
|
|
183
|
+
|
|
184
|
+
operation = registry.get(step.op)
|
|
185
|
+
result = operation.run(step.params, ctx)
|
|
186
|
+
report.results.append(result)
|
|
187
|
+
|
|
188
|
+
if not result.ok and not continue_on_error:
|
|
189
|
+
report.stopped_at = step.label
|
|
190
|
+
break
|
|
191
|
+
|
|
192
|
+
return report
|
data_factory/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""The hosted HTTP service.
|
|
2
|
+
|
|
3
|
+
Imported lazily so that ``import data_factory`` never requires FastAPI. Install
|
|
4
|
+
the extra to use it::
|
|
5
|
+
|
|
6
|
+
pip install "data-factory[web]"
|
|
7
|
+
uvicorn data_factory.service.app:app --port 8000
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
__all__ = ["app", "binding", "jobs", "workspace"]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def __getattr__(name: str) -> Any:
|
|
18
|
+
if name == "app":
|
|
19
|
+
from .app import app
|
|
20
|
+
|
|
21
|
+
return app
|
|
22
|
+
if name in ("binding", "jobs", "workspace"):
|
|
23
|
+
import importlib
|
|
24
|
+
|
|
25
|
+
return importlib.import_module(f".{name}", __name__)
|
|
26
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
@@ -0,0 +1,483 @@
|
|
|
1
|
+
"""The hosted HTTP service.
|
|
2
|
+
|
|
3
|
+
A deployable API rather than the illustrative one in ``examples/``. The
|
|
4
|
+
difference is entirely about trust: this one never accepts a filesystem path,
|
|
5
|
+
sandboxes every dataset, caps uploads, expires old work, and keeps tracebacks
|
|
6
|
+
out of responses.
|
|
7
|
+
|
|
8
|
+
Run it::
|
|
9
|
+
|
|
10
|
+
pip install "data-factory[web]"
|
|
11
|
+
uvicorn data_factory.service.app:app --host 0.0.0.0 --port 8000
|
|
12
|
+
|
|
13
|
+
Configuration, all optional, all environment variables::
|
|
14
|
+
|
|
15
|
+
DATAFACTORY_WORKSPACE where datasets live (default ~/.datafactory/workspaces)
|
|
16
|
+
DATAFACTORY_TTL_HOURS how long an anonymous dataset survives (default 24)
|
|
17
|
+
DATAFACTORY_ORIGINS comma-separated CORS origins (default: same-origin only)
|
|
18
|
+
DATAFACTORY_API_KEY if set, every /api call must send X-API-Key
|
|
19
|
+
DATAFACTORY_MAX_UPLOAD_MB overrides the upload cap
|
|
20
|
+
|
|
21
|
+
The endpoints are shaped for the flow a browser actually performs: upload a
|
|
22
|
+
zip, look at what it is, run steps against it, download the result.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import os
|
|
28
|
+
import secrets
|
|
29
|
+
import shutil
|
|
30
|
+
import tempfile
|
|
31
|
+
import time
|
|
32
|
+
from contextlib import asynccontextmanager
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
from typing import Any
|
|
35
|
+
|
|
36
|
+
from fastapi import Depends, FastAPI, File, Header, HTTPException, UploadFile
|
|
37
|
+
from fastapi.middleware.cors import CORSMiddleware
|
|
38
|
+
from fastapi.responses import FileResponse
|
|
39
|
+
from fastapi.staticfiles import StaticFiles
|
|
40
|
+
from starlette.concurrency import run_in_threadpool
|
|
41
|
+
from pydantic import BaseModel, Field
|
|
42
|
+
|
|
43
|
+
import data_factory as df
|
|
44
|
+
|
|
45
|
+
from ..core import registry
|
|
46
|
+
from ..core.annotations import formats
|
|
47
|
+
from ..core.annotations.model import AnnotationError
|
|
48
|
+
from ..core.registry import ParamError
|
|
49
|
+
from . import binding
|
|
50
|
+
from . import jobs as jobs_module
|
|
51
|
+
from .limits import BodyLimit
|
|
52
|
+
from .routers import annotate as annotate_router
|
|
53
|
+
from .routers import local as local_router
|
|
54
|
+
from .routers import media as media_router
|
|
55
|
+
from .routers import predictions as predictions_router
|
|
56
|
+
from .routers import promote as promote_router
|
|
57
|
+
from .routers import tools as tools_router
|
|
58
|
+
from .workspace import (
|
|
59
|
+
MAX_UPLOAD_BYTES,
|
|
60
|
+
Workspace,
|
|
61
|
+
WorkspaceError,
|
|
62
|
+
extract_zip,
|
|
63
|
+
make_zip,
|
|
64
|
+
resolve_workspace_root,
|
|
65
|
+
)
|
|
66
|
+
from ..annotate import images as images_mod
|
|
67
|
+
from ..annotate import projects as projects_mod
|
|
68
|
+
from ..annotate.store import Store, StoreError
|
|
69
|
+
from .deps import get_store, principal, set_jobs_manager, store_path
|
|
70
|
+
|
|
71
|
+
WORKSPACE = Workspace(
|
|
72
|
+
resolve_workspace_root(),
|
|
73
|
+
ttl_seconds=int(float(os.environ.get("DATAFACTORY_TTL_HOURS", "24")) * 3600),
|
|
74
|
+
)
|
|
75
|
+
JOBS = jobs_module.JobManager()
|
|
76
|
+
set_jobs_manager(JOBS)
|
|
77
|
+
MAX_UPLOAD = int(os.environ.get("DATAFACTORY_MAX_UPLOAD_MB", "0")) * 1024 * 1024 or MAX_UPLOAD_BYTES
|
|
78
|
+
API_KEY = os.environ.get("DATAFACTORY_API_KEY", "")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@asynccontextmanager
|
|
82
|
+
async def lifespan(app: FastAPI):
|
|
83
|
+
# One sweep at boot clears whatever a previous container left behind; a
|
|
84
|
+
# real deployment should also run it on a timer or a cron sidecar.
|
|
85
|
+
WORKSPACE.sweep()
|
|
86
|
+
yield
|
|
87
|
+
JOBS.shutdown()
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
app = FastAPI(
|
|
91
|
+
title="EasyAnnotate",
|
|
92
|
+
version=df.__version__,
|
|
93
|
+
summary="Dataset preparation for computer vision: convert, clean, augment, analyse.",
|
|
94
|
+
lifespan=lifespan,
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
# Annotation is stateful CRUD with a latency budget and cannot be expressed as
|
|
98
|
+
# an operation, so it gets routers of its own rather than being forced through
|
|
99
|
+
# the registry. See data_factory/annotate/__init__.py for the reasoning.
|
|
100
|
+
app.include_router(annotate_router.router)
|
|
101
|
+
app.include_router(media_router.router)
|
|
102
|
+
app.include_router(promote_router.router)
|
|
103
|
+
app.include_router(predictions_router.router)
|
|
104
|
+
app.include_router(tools_router.router)
|
|
105
|
+
app.include_router(local_router.router)
|
|
106
|
+
|
|
107
|
+
# Before anything reads a body. See limits.py for why the handler's own byte
|
|
108
|
+
# count could never do this.
|
|
109
|
+
app.add_middleware(BodyLimit, limit=MAX_UPLOAD)
|
|
110
|
+
# Outermost: a request that names another host never reaches a route.
|
|
111
|
+
app.add_middleware(local_router.LocalHostFence)
|
|
112
|
+
|
|
113
|
+
_origins = [o.strip() for o in os.environ.get("DATAFACTORY_ORIGINS", "").split(",") if o.strip()]
|
|
114
|
+
if _origins:
|
|
115
|
+
app.add_middleware(
|
|
116
|
+
CORSMiddleware,
|
|
117
|
+
allow_origins=_origins,
|
|
118
|
+
# PUT and PATCH are how the editor saves shapes and changes state.
|
|
119
|
+
allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE"],
|
|
120
|
+
allow_headers=["X-API-Key", "Content-Type"],
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
async def require_key(x_api_key: str = Header(default="")) -> None:
|
|
125
|
+
"""Optional shared-secret gate.
|
|
126
|
+
|
|
127
|
+
Off by default so the tool runs locally with no ceremony. Set
|
|
128
|
+
DATAFACTORY_API_KEY before exposing it to the internet -- without it,
|
|
129
|
+
anyone can spend your CPU and disk.
|
|
130
|
+
"""
|
|
131
|
+
if not API_KEY:
|
|
132
|
+
return
|
|
133
|
+
if not x_api_key.isascii() or not secrets.compare_digest(x_api_key, API_KEY):
|
|
134
|
+
raise HTTPException(401, "invalid or missing X-API-Key")
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
# ----------------------------------------------------------------- models
|
|
138
|
+
|
|
139
|
+
class RunRequest(BaseModel):
|
|
140
|
+
params: dict[str, Any] = Field(default_factory=dict)
|
|
141
|
+
input: str | None = Field(default=None, description="'source', or an earlier output name")
|
|
142
|
+
output_name: str = Field(default="", description="what to call the result")
|
|
143
|
+
dry_run: bool = False
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
# ----------------------------------------------------------------- meta
|
|
147
|
+
|
|
148
|
+
@app.get("/api/health")
|
|
149
|
+
def health() -> dict[str, Any]:
|
|
150
|
+
return {"ok": True, "version": df.__version__, "operations": len(registry.all_operations())}
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@app.get("/api/operations", dependencies=[Depends(require_key)])
|
|
154
|
+
def list_operations() -> dict[str, Any]:
|
|
155
|
+
"""Every operation, described so a front-end can generate its own forms.
|
|
156
|
+
|
|
157
|
+
Path parameters are omitted: the client cannot set them, so rendering an
|
|
158
|
+
input for one would only produce a request the service refuses.
|
|
159
|
+
"""
|
|
160
|
+
return binding.public_schema()
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
@app.get("/api/operations/{name}", dependencies=[Depends(require_key)])
|
|
164
|
+
def describe_operation(name: str) -> dict[str, Any]:
|
|
165
|
+
try:
|
|
166
|
+
return binding.describe_operation(registry.get(name))
|
|
167
|
+
except KeyError:
|
|
168
|
+
raise HTTPException(404, f"unknown operation {name!r}") from None
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
# ----------------------------------------------------------------- datasets
|
|
172
|
+
|
|
173
|
+
@app.post("/api/datasets", dependencies=[Depends(require_key)], status_code=201)
|
|
174
|
+
async def upload_dataset(file: UploadFile = File(...)) -> dict[str, Any]:
|
|
175
|
+
"""Upload a zipped dataset and get back an opaque id.
|
|
176
|
+
|
|
177
|
+
The id is the only handle a client ever holds; it never learns a path.
|
|
178
|
+
"""
|
|
179
|
+
if not (file.filename or "").lower().endswith(".zip"):
|
|
180
|
+
raise HTTPException(415, "upload a .zip archive of your dataset")
|
|
181
|
+
|
|
182
|
+
dataset = WORKSPACE.create()
|
|
183
|
+
staged = Path(tempfile.mkdtemp()) / "upload.zip"
|
|
184
|
+
written = 0
|
|
185
|
+
try:
|
|
186
|
+
with staged.open("wb") as handle:
|
|
187
|
+
while chunk := await file.read(1 << 20):
|
|
188
|
+
written += len(chunk)
|
|
189
|
+
if written > MAX_UPLOAD:
|
|
190
|
+
raise HTTPException(
|
|
191
|
+
413, f"upload exceeds the {MAX_UPLOAD / 1e6:.0f} MB limit"
|
|
192
|
+
)
|
|
193
|
+
handle.write(chunk)
|
|
194
|
+
# Off the event loop: unpacking a large archive is seconds of disk
|
|
195
|
+
# work, and on the loop it would stall every save in the editor.
|
|
196
|
+
stats = await run_in_threadpool(extract_zip, staged, dataset.source, workspace=WORKSPACE)
|
|
197
|
+
except HTTPException:
|
|
198
|
+
WORKSPACE.delete(dataset.id)
|
|
199
|
+
raise
|
|
200
|
+
except WorkspaceError as exc:
|
|
201
|
+
WORKSPACE.delete(dataset.id)
|
|
202
|
+
raise HTTPException(400, str(exc)) from None
|
|
203
|
+
finally:
|
|
204
|
+
shutil.rmtree(staged.parent, ignore_errors=True)
|
|
205
|
+
|
|
206
|
+
return {"id": dataset.id, "uploaded": stats, **(await run_in_threadpool(_describe, dataset))}
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
@app.get("/api/datasets/{dataset_id}", dependencies=[Depends(require_key)])
|
|
210
|
+
def dataset_info(dataset_id: str) -> dict[str, Any]:
|
|
211
|
+
return {"id": dataset_id, **_describe(_dataset(dataset_id))}
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
@app.delete("/api/datasets/{dataset_id}", dependencies=[Depends(require_key)], status_code=204)
|
|
215
|
+
def delete_dataset(dataset_id: str) -> None:
|
|
216
|
+
_dataset(dataset_id)
|
|
217
|
+
WORKSPACE.delete(dataset_id)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
@app.get("/api/datasets/{dataset_id}/download", dependencies=[Depends(require_key)])
|
|
221
|
+
def download(dataset_id: str, output: str = "") -> FileResponse:
|
|
222
|
+
"""Zip an output (or the source) for the browser to save."""
|
|
223
|
+
dataset = _dataset(dataset_id)
|
|
224
|
+
try:
|
|
225
|
+
source = binding.resolve_input(dataset, output or "source", WORKSPACE)
|
|
226
|
+
archive = make_zip(source, dataset.root / "download" / f"{output or 'source'}.zip",
|
|
227
|
+
workspace=WORKSPACE)
|
|
228
|
+
except (WorkspaceError, binding.BindingError) as exc:
|
|
229
|
+
raise HTTPException(404, str(exc)) from None
|
|
230
|
+
return FileResponse(
|
|
231
|
+
archive,
|
|
232
|
+
media_type="application/zip",
|
|
233
|
+
filename=f"{output or 'dataset'}-{dataset_id[:8]}.zip",
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
# ----------------------------------------------------------------- running
|
|
238
|
+
|
|
239
|
+
@app.post("/api/datasets/{dataset_id}/run/{name}", dependencies=[Depends(require_key)])
|
|
240
|
+
def run_operation(dataset_id: str, name: str, request: RunRequest) -> dict[str, Any]:
|
|
241
|
+
"""Start an operation. Returns a job id immediately.
|
|
242
|
+
|
|
243
|
+
Validation happens synchronously so a bad request fails now, with the same
|
|
244
|
+
message the CLI would print, rather than as a failed job thirty seconds
|
|
245
|
+
later.
|
|
246
|
+
"""
|
|
247
|
+
dataset = _dataset(dataset_id)
|
|
248
|
+
try:
|
|
249
|
+
operation = registry.get(name)
|
|
250
|
+
except KeyError:
|
|
251
|
+
raise HTTPException(404, f"unknown operation {name!r}") from None
|
|
252
|
+
|
|
253
|
+
try:
|
|
254
|
+
bound, output_dir = binding.bind(
|
|
255
|
+
operation,
|
|
256
|
+
request.params,
|
|
257
|
+
dataset=dataset,
|
|
258
|
+
workspace=WORKSPACE,
|
|
259
|
+
input_ref=request.input,
|
|
260
|
+
output_name=request.output_name,
|
|
261
|
+
)
|
|
262
|
+
except binding.BindingError as exc:
|
|
263
|
+
raise HTTPException(400, str(exc)) from None
|
|
264
|
+
except ParamError as exc:
|
|
265
|
+
raise HTTPException(422, str(exc)) from None
|
|
266
|
+
except WorkspaceError as exc:
|
|
267
|
+
raise HTTPException(400, str(exc)) from None
|
|
268
|
+
|
|
269
|
+
dry_run = request.dry_run
|
|
270
|
+
job_id = secrets.token_hex(8)
|
|
271
|
+
|
|
272
|
+
def work(context):
|
|
273
|
+
context.dry_run = dry_run
|
|
274
|
+
result = operation.func(bound, context)
|
|
275
|
+
# Provenance is recorded by Operation.run, which we bypass to keep the
|
|
276
|
+
# already-bound parameters, so record it explicitly here.
|
|
277
|
+
_record(operation.name, bound, result)
|
|
278
|
+
return result
|
|
279
|
+
|
|
280
|
+
# A run creates or changes an output, so the cached description is stale
|
|
281
|
+
# the moment it is submitted.
|
|
282
|
+
_invalidate(dataset_id)
|
|
283
|
+
|
|
284
|
+
job = JOBS.submit(
|
|
285
|
+
job_id,
|
|
286
|
+
operation.name,
|
|
287
|
+
dataset_id,
|
|
288
|
+
work,
|
|
289
|
+
output_name=output_dir.name if output_dir else "",
|
|
290
|
+
)
|
|
291
|
+
return job.as_dict()
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
class ExportRequest(BaseModel):
|
|
295
|
+
to: str = "yolo"
|
|
296
|
+
states: list[str] = Field(default_factory=lambda: ["reviewed"])
|
|
297
|
+
copy_images: bool = True
|
|
298
|
+
segmentation: bool = True
|
|
299
|
+
include_empty: bool = True
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
@app.post("/api/projects/{project_id}/export", dependencies=[Depends(require_key)])
|
|
303
|
+
def export_project(
|
|
304
|
+
project_id: str,
|
|
305
|
+
request: ExportRequest,
|
|
306
|
+
store: Store = Depends(get_store),
|
|
307
|
+
owner: str = Depends(principal),
|
|
308
|
+
) -> dict[str, Any]:
|
|
309
|
+
"""Export a project as a dataset the browser can download.
|
|
310
|
+
|
|
311
|
+
The one bridge from the annotation world back to the batch world: the
|
|
312
|
+
same annotate.export operation the CLI runs, as a job, into a throwaway
|
|
313
|
+
workspace sandbox, with the download URL handed back up front. Nothing in
|
|
314
|
+
the request is a path -- the database, the output folder and the image
|
|
315
|
+
root are all resolved here, which is what lets this exist on the hosted
|
|
316
|
+
service where no client may ever name a file.
|
|
317
|
+
|
|
318
|
+
Without this a hosted user could annotate and never get their work out.
|
|
319
|
+
"""
|
|
320
|
+
try:
|
|
321
|
+
project = projects_mod.get(store, project_id, owner)
|
|
322
|
+
except StoreError as exc:
|
|
323
|
+
raise HTTPException(404, str(exc)) from None
|
|
324
|
+
if request.to not in formats.FORMATS:
|
|
325
|
+
raise HTTPException(400, f"unknown format {request.to!r}; known: {', '.join(formats.FORMATS)}")
|
|
326
|
+
unknown = [s for s in request.states if s not in images_mod.STATES]
|
|
327
|
+
if unknown or not request.states:
|
|
328
|
+
raise HTTPException(
|
|
329
|
+
400, f"states must be some of {', '.join(images_mod.STATES)}; got {request.states!r}"
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
operation = registry.get("annotate.export")
|
|
333
|
+
sandbox = WORKSPACE.create()
|
|
334
|
+
output_dir = sandbox.output("export")
|
|
335
|
+
bound: dict[str, Any] = {param.name: param.default for param in operation.params}
|
|
336
|
+
bound.update(
|
|
337
|
+
{
|
|
338
|
+
"database": store_path(),
|
|
339
|
+
"project": project.id,
|
|
340
|
+
"output": output_dir,
|
|
341
|
+
"to": request.to,
|
|
342
|
+
"states": list(request.states),
|
|
343
|
+
"owner": owner,
|
|
344
|
+
"copy_images": request.copy_images,
|
|
345
|
+
"segmentation": request.segmentation,
|
|
346
|
+
"include_empty": request.include_empty,
|
|
347
|
+
}
|
|
348
|
+
)
|
|
349
|
+
|
|
350
|
+
job_id = secrets.token_hex(8)
|
|
351
|
+
|
|
352
|
+
def work(context):
|
|
353
|
+
result = operation.func(bound, context)
|
|
354
|
+
_record(operation.name, bound, result)
|
|
355
|
+
return result
|
|
356
|
+
|
|
357
|
+
job = JOBS.submit(job_id, operation.name, sandbox.id, work, output_name="export")
|
|
358
|
+
return {
|
|
359
|
+
**job.as_dict(),
|
|
360
|
+
"dataset": sandbox.id,
|
|
361
|
+
"download": f"/api/datasets/{sandbox.id}/download?output=export",
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
@app.get("/api/jobs/{job_id}", dependencies=[Depends(require_key)])
|
|
366
|
+
def job_status(job_id: str) -> dict[str, Any]:
|
|
367
|
+
job = JOBS.get(job_id)
|
|
368
|
+
if job is None:
|
|
369
|
+
raise HTTPException(404, "unknown job")
|
|
370
|
+
return job.as_dict()
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
@app.post("/api/jobs/{job_id}/cancel", dependencies=[Depends(require_key)])
|
|
374
|
+
def cancel_job(job_id: str) -> dict[str, Any]:
|
|
375
|
+
"""Cooperative cancellation: the run stops at its next checkpoint."""
|
|
376
|
+
if not JOBS.cancel(job_id):
|
|
377
|
+
raise HTTPException(409, "that job has already finished")
|
|
378
|
+
return {"id": job_id, "state": "cancelling"}
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
@app.get("/api/datasets/{dataset_id}/jobs", dependencies=[Depends(require_key)])
|
|
382
|
+
def dataset_jobs(dataset_id: str) -> dict[str, Any]:
|
|
383
|
+
_dataset(dataset_id)
|
|
384
|
+
return {"jobs": [j.as_dict() for j in JOBS.for_dataset(dataset_id)]}
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
# ----------------------------------------------------------------- helpers
|
|
388
|
+
|
|
389
|
+
STATIC = Path(__file__).parent / "static"
|
|
390
|
+
|
|
391
|
+
# Only this package's own files are served here -- no user content ever lands
|
|
392
|
+
# in this directory, so there is nothing for a traversal to reach that is not
|
|
393
|
+
# already public.
|
|
394
|
+
app.mount("/static", StaticFiles(directory=STATIC), name="static")
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
@app.get("/annotate", include_in_schema=False)
|
|
398
|
+
def annotate_page() -> FileResponse:
|
|
399
|
+
"""The editor.
|
|
400
|
+
|
|
401
|
+
Hand-written ES modules with no build step, so the wheel stays
|
|
402
|
+
self-contained and ``pip install`` keeps working offline. Phase 0 measured
|
|
403
|
+
a plain canvas holding 60fps with 300 shapes on a 4000x3000 image, which is
|
|
404
|
+
what makes that defensible rather than merely convenient.
|
|
405
|
+
"""
|
|
406
|
+
return FileResponse(STATIC / "annotate.html", media_type="text/html")
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
@app.get("/", include_in_schema=False)
|
|
410
|
+
def index() -> FileResponse:
|
|
411
|
+
"""The built-in UI.
|
|
412
|
+
|
|
413
|
+
It builds its own forms from ``/api/operations``, so an operation added to
|
|
414
|
+
the Python package appears in the browser with no front-end change -- the
|
|
415
|
+
same generation the CLI and the menu get.
|
|
416
|
+
"""
|
|
417
|
+
return FileResponse(STATIC / "index.html", media_type="text/html")
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _dataset(dataset_id: str):
|
|
421
|
+
try:
|
|
422
|
+
return WORKSPACE.get(dataset_id)
|
|
423
|
+
except WorkspaceError as exc:
|
|
424
|
+
raise HTTPException(404, str(exc)) from None
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
#: ``_describe`` costs a full tree walk plus a complete annotation read, which
|
|
428
|
+
#: is O(dataset) per request. That is affordable for a page load and hopeless
|
|
429
|
+
#: for anything that polls, so the answer is held briefly and dropped whenever
|
|
430
|
+
#: a run could have changed it.
|
|
431
|
+
_DESCRIBE_TTL = 10.0
|
|
432
|
+
_describe_cache: dict[str, tuple[float, dict[str, Any]]] = {}
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
def _invalidate(dataset_id: str) -> None:
|
|
436
|
+
_describe_cache.pop(dataset_id, None)
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def _describe(dataset) -> dict[str, Any]:
|
|
440
|
+
"""What the dataset is, as far as we can tell, for the UI to show."""
|
|
441
|
+
cached = _describe_cache.get(dataset.id)
|
|
442
|
+
now = time.time()
|
|
443
|
+
if cached is not None and now - cached[0] < _DESCRIBE_TTL:
|
|
444
|
+
# Recompute the countdown; everything else is stable for the window.
|
|
445
|
+
payload = dict(cached[1])
|
|
446
|
+
payload["expires_in_seconds"] = max(
|
|
447
|
+
0, int(dataset.created_at() + WORKSPACE.ttl_seconds - now)
|
|
448
|
+
)
|
|
449
|
+
return payload
|
|
450
|
+
payload = _build_description(dataset)
|
|
451
|
+
_describe_cache[dataset.id] = (now, payload)
|
|
452
|
+
return dict(payload)
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def _build_description(dataset) -> dict[str, Any]:
|
|
456
|
+
payload: dict[str, Any] = {
|
|
457
|
+
"inputs": binding.available_inputs(dataset),
|
|
458
|
+
"size_bytes": dataset.size_bytes(),
|
|
459
|
+
"expires_in_seconds": max(
|
|
460
|
+
0, int(dataset.created_at() + WORKSPACE.ttl_seconds - time.time())
|
|
461
|
+
),
|
|
462
|
+
}
|
|
463
|
+
try:
|
|
464
|
+
loaded = formats.read_dataset(dataset.source, "auto", recursive=True)
|
|
465
|
+
payload["format"] = loaded.source_format
|
|
466
|
+
payload["stats"] = loaded.stats()
|
|
467
|
+
payload["classes"] = loaded.labels
|
|
468
|
+
payload["warnings"] = loaded.warnings[:10]
|
|
469
|
+
except AnnotationError as exc:
|
|
470
|
+
# Not fatal: plenty of useful operations work on a folder of images
|
|
471
|
+
# with no annotations at all.
|
|
472
|
+
payload["format"] = None
|
|
473
|
+
payload["detection_error"] = str(exc)
|
|
474
|
+
return payload
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def _record(name: str, bound: dict[str, Any], result) -> None:
|
|
478
|
+
try:
|
|
479
|
+
from ..core import provenance
|
|
480
|
+
|
|
481
|
+
provenance.record(name, bound, result, version=df.__version__)
|
|
482
|
+
except Exception:
|
|
483
|
+
pass
|