easyannotate 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. data_factory/__init__.py +82 -0
  2. data_factory/__main__.py +6 -0
  3. data_factory/annotate/__init__.py +15 -0
  4. data_factory/annotate/attach.py +398 -0
  5. data_factory/annotate/demo.py +161 -0
  6. data_factory/annotate/images.py +443 -0
  7. data_factory/annotate/labels.py +399 -0
  8. data_factory/annotate/loop.py +359 -0
  9. data_factory/annotate/projects.py +207 -0
  10. data_factory/annotate/review.py +471 -0
  11. data_factory/annotate/schema.sql +171 -0
  12. data_factory/annotate/shapes.py +488 -0
  13. data_factory/annotate/store.py +292 -0
  14. data_factory/annotate/tools/__init__.py +39 -0
  15. data_factory/annotate/tools/fake.py +170 -0
  16. data_factory/annotate/tools/install.py +254 -0
  17. data_factory/annotate/tools/local.py +389 -0
  18. data_factory/annotate/tools/postprocess.py +175 -0
  19. data_factory/annotate/tools/protocol.py +238 -0
  20. data_factory/annotate/tools/registry.py +196 -0
  21. data_factory/annotate/tools/results.py +83 -0
  22. data_factory/annotate/tools/serve.py +106 -0
  23. data_factory/cli/__init__.py +21 -0
  24. data_factory/cli/main.py +837 -0
  25. data_factory/cli/menu.py +173 -0
  26. data_factory/cli/render.py +158 -0
  27. data_factory/core/__init__.py +11 -0
  28. data_factory/core/annotations/__init__.py +5 -0
  29. data_factory/core/annotations/coco.py +394 -0
  30. data_factory/core/annotations/csvfmt.py +174 -0
  31. data_factory/core/annotations/dota.py +98 -0
  32. data_factory/core/annotations/formats.py +626 -0
  33. data_factory/core/annotations/labelme.py +153 -0
  34. data_factory/core/annotations/model.py +334 -0
  35. data_factory/core/annotations/voc.py +231 -0
  36. data_factory/core/annotations/yolo.py +283 -0
  37. data_factory/core/bktree.py +89 -0
  38. data_factory/core/fs.py +245 -0
  39. data_factory/core/imaging.py +626 -0
  40. data_factory/core/ops/__init__.py +44 -0
  41. data_factory/core/ops/analyze.py +383 -0
  42. data_factory/core/ops/annotate_export.py +326 -0
  43. data_factory/core/ops/audit.py +274 -0
  44. data_factory/core/ops/augment.py +454 -0
  45. data_factory/core/ops/collect.py +241 -0
  46. data_factory/core/ops/combine.py +582 -0
  47. data_factory/core/ops/convert.py +212 -0
  48. data_factory/core/ops/dataset.py +246 -0
  49. data_factory/core/ops/dedup.py +235 -0
  50. data_factory/core/ops/export.py +168 -0
  51. data_factory/core/ops/image.py +407 -0
  52. data_factory/core/ops/labels.py +215 -0
  53. data_factory/core/ops/pairing.py +133 -0
  54. data_factory/core/ops/rename.py +154 -0
  55. data_factory/core/ops/report.py +323 -0
  56. data_factory/core/ops/visualize.py +164 -0
  57. data_factory/core/pairs.py +144 -0
  58. data_factory/core/parallel.py +191 -0
  59. data_factory/core/provenance.py +136 -0
  60. data_factory/core/registry.py +339 -0
  61. data_factory/core/stats.py +403 -0
  62. data_factory/core/types.py +148 -0
  63. data_factory/core/writer.py +194 -0
  64. data_factory/desktop.py +207 -0
  65. data_factory/hardware.py +243 -0
  66. data_factory/pipeline.py +192 -0
  67. data_factory/py.typed +0 -0
  68. data_factory/service/__init__.py +26 -0
  69. data_factory/service/app.py +483 -0
  70. data_factory/service/binding.py +174 -0
  71. data_factory/service/deps.py +133 -0
  72. data_factory/service/jobs.py +177 -0
  73. data_factory/service/limits.py +71 -0
  74. data_factory/service/routers/__init__.py +11 -0
  75. data_factory/service/routers/annotate.py +582 -0
  76. data_factory/service/routers/local.py +464 -0
  77. data_factory/service/routers/media.py +154 -0
  78. data_factory/service/routers/predictions.py +116 -0
  79. data_factory/service/routers/promote.py +182 -0
  80. data_factory/service/routers/tools.py +308 -0
  81. data_factory/service/static/annotate/api.js +246 -0
  82. data_factory/service/static/annotate/app.js +1696 -0
  83. data_factory/service/static/annotate/assist.js +106 -0
  84. data_factory/service/static/annotate/filmstrip.js +286 -0
  85. data_factory/service/static/annotate/hit.js +146 -0
  86. data_factory/service/static/annotate/labels.js +197 -0
  87. data_factory/service/static/annotate/render.js +286 -0
  88. data_factory/service/static/annotate/scene.js +277 -0
  89. data_factory/service/static/annotate/shapelist.js +72 -0
  90. data_factory/service/static/annotate/tools/box.js +96 -0
  91. data_factory/service/static/annotate/tools/line.js +96 -0
  92. data_factory/service/static/annotate/tools/obb.js +88 -0
  93. data_factory/service/static/annotate/tools/point.js +49 -0
  94. data_factory/service/static/annotate/tools/polygon.js +110 -0
  95. data_factory/service/static/annotate/tools/select.js +227 -0
  96. data_factory/service/static/annotate/undo.js +176 -0
  97. data_factory/service/static/annotate/viewport.js +156 -0
  98. data_factory/service/static/annotate.html +390 -0
  99. data_factory/service/static/index.html +632 -0
  100. data_factory/service/workspace.py +244 -0
  101. easyannotate/__init__.py +64 -0
  102. easyannotate/__main__.py +5 -0
  103. easyannotate/project.py +181 -0
  104. easyannotate/py.typed +0 -0
  105. easyannotate-0.4.0.dist-info/METADATA +767 -0
  106. easyannotate-0.4.0.dist-info/RECORD +109 -0
  107. easyannotate-0.4.0.dist-info/WHEEL +4 -0
  108. easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
  109. easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,192 @@
1
+ """Run a recipe: an ordered list of operations described in YAML or JSON.
2
+
3
+ A dataset build is rarely one command. Writing the sequence down makes it
4
+ reproducible and reviewable, and means the same build can be re-run later, by
5
+ someone else, or from a web backend, without anyone remembering the order.
6
+
7
+ name: door-v4
8
+ vars:
9
+ root: D:/Data_factory/Projects/Project_Door/working
10
+ steps:
11
+ - op: clean.orphans
12
+ with: {folder: "${root}", scope: labels, action: delete}
13
+ - op: labels.rename
14
+ with: {folder: "${root}", fold_case: true}
15
+ - op: dataset.split
16
+ with: {folder: "${root}", output: "${root}/../splits"}
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import json
22
+ from dataclasses import dataclass, field
23
+ from pathlib import Path
24
+ from typing import Any
25
+
26
+ from .core import registry
27
+ from .core.types import Context, OpResult
28
+
29
+
30
+ class RecipeError(ValueError):
31
+ """Raised when a recipe file is malformed or names an unknown operation."""
32
+
33
+
34
+ @dataclass
35
+ class Step:
36
+ op: str
37
+ params: dict[str, Any] = field(default_factory=dict)
38
+ name: str = ""
39
+ skip: bool = False
40
+
41
+ @property
42
+ def label(self) -> str:
43
+ return self.name or self.op
44
+
45
+
46
+ @dataclass
47
+ class Recipe:
48
+ name: str
49
+ steps: list[Step]
50
+ description: str = ""
51
+ variables: dict[str, str] = field(default_factory=dict)
52
+
53
+ @classmethod
54
+ def load(cls, path: Path | str, overrides: dict[str, str] | None = None) -> Recipe:
55
+ # Accept a plain string too: the CLI always hands over a Path, but a
56
+ # library caller reasonably writes Recipe.load("build.yaml").
57
+ path = Path(path)
58
+ document = _load_document(path)
59
+ return cls.from_dict(document, source=path, overrides=overrides)
60
+
61
+ @classmethod
62
+ def from_dict(
63
+ cls,
64
+ document: dict[str, Any],
65
+ source: Path | None = None,
66
+ overrides: dict[str, str] | None = None,
67
+ ) -> Recipe:
68
+ if not isinstance(document, dict):
69
+ raise RecipeError("a recipe must be a mapping at the top level")
70
+ raw_steps = document.get("steps")
71
+ if not isinstance(raw_steps, list) or not raw_steps:
72
+ raise RecipeError("a recipe needs a non-empty 'steps' list")
73
+
74
+ # Overrides win over the file's own vars, so one recipe can be pointed
75
+ # at a different dataset without being edited.
76
+ variables = {str(k): str(v) for k, v in (document.get("vars") or {}).items()}
77
+ variables.update({str(k): str(v) for k, v in (overrides or {}).items()})
78
+ steps: list[Step] = []
79
+ for position, entry in enumerate(raw_steps, start=1):
80
+ if not isinstance(entry, dict) or "op" not in entry:
81
+ raise RecipeError(f"step {position}: each step needs an 'op' key")
82
+ op_name = str(entry["op"])
83
+ try:
84
+ operation = registry.get(op_name)
85
+ except KeyError as exc:
86
+ raise RecipeError(f"step {position}: {exc}") from None
87
+
88
+ params = entry.get("with") or entry.get("params") or {}
89
+ if not isinstance(params, dict):
90
+ raise RecipeError(f"step {position} ({op_name}): 'with' must be a mapping")
91
+ params = {k: _substitute(v, variables) for k, v in params.items()}
92
+
93
+ # Validate now rather than three steps into a long run -- but do not
94
+ # check that paths exist, since a step's input is often the previous
95
+ # step's output and will not be there until the run reaches it.
96
+ try:
97
+ operation.bind(params, check_paths=False)
98
+ except registry.ParamError as exc:
99
+ raise RecipeError(f"step {position} ({op_name}): {exc}") from None
100
+
101
+ steps.append(
102
+ Step(op=op_name, params=params, name=str(entry.get("name", "")),
103
+ skip=bool(entry.get("skip", False)))
104
+ )
105
+
106
+ return cls(
107
+ name=str(document.get("name") or (source.stem if source else "recipe")),
108
+ description=str(document.get("description", "")),
109
+ variables=variables,
110
+ steps=steps,
111
+ )
112
+
113
+
114
+ def _substitute(value: Any, variables: dict[str, str]) -> Any:
115
+ """Expand ``${name}`` references, recursing through lists and mappings."""
116
+ if isinstance(value, str):
117
+ for key, replacement in variables.items():
118
+ value = value.replace("${" + key + "}", replacement)
119
+ return value
120
+ if isinstance(value, list):
121
+ return [_substitute(v, variables) for v in value]
122
+ if isinstance(value, dict):
123
+ return {k: _substitute(v, variables) for k, v in value.items()}
124
+ return value
125
+
126
+
127
+ def _load_document(path: Path) -> dict[str, Any]:
128
+ text = path.read_text(encoding="utf-8")
129
+ if path.suffix.lower() in (".yaml", ".yml"):
130
+ try:
131
+ import yaml
132
+ except ImportError as exc:
133
+ raise RecipeError(
134
+ f"reading {path.name} needs PyYAML installed, or write the recipe as .json ({exc})"
135
+ ) from None
136
+ return yaml.safe_load(text) or {}
137
+ return json.loads(text)
138
+
139
+
140
+ @dataclass
141
+ class RunReport:
142
+ """Outcome of a whole recipe, one entry per executed step."""
143
+
144
+ recipe: str
145
+ results: list[OpResult] = field(default_factory=list)
146
+ stopped_at: str | None = None
147
+
148
+ @property
149
+ def ok(self) -> bool:
150
+ return self.stopped_at is None and all(r.ok for r in self.results)
151
+
152
+ def as_dict(self) -> dict[str, Any]:
153
+ return {
154
+ "recipe": self.recipe,
155
+ "ok": self.ok,
156
+ "stopped_at": self.stopped_at,
157
+ "steps": [r.as_dict(include_actions=False) for r in self.results],
158
+ }
159
+
160
+
161
+ def run(
162
+ recipe: Recipe,
163
+ ctx: Context | None = None,
164
+ *,
165
+ continue_on_error: bool = False,
166
+ on_step: Any = None,
167
+ ) -> RunReport:
168
+ """Execute every step in order.
169
+
170
+ Stops at the first failure by default: later steps usually assume the
171
+ earlier ones succeeded, and carrying on with a half-cleaned folder does more
172
+ damage than stopping.
173
+ """
174
+ ctx = ctx or Context()
175
+ report = RunReport(recipe=recipe.name)
176
+
177
+ for position, step in enumerate(recipe.steps, start=1):
178
+ if step.skip:
179
+ continue
180
+ ctx.check_cancelled()
181
+ if on_step is not None:
182
+ on_step(position, len(recipe.steps), step)
183
+
184
+ operation = registry.get(step.op)
185
+ result = operation.run(step.params, ctx)
186
+ report.results.append(result)
187
+
188
+ if not result.ok and not continue_on_error:
189
+ report.stopped_at = step.label
190
+ break
191
+
192
+ return report
data_factory/py.typed ADDED
File without changes
@@ -0,0 +1,26 @@
1
+ """The hosted HTTP service.
2
+
3
+ Imported lazily so that ``import data_factory`` never requires FastAPI. Install
4
+ the extra to use it::
5
+
6
+ pip install "data-factory[web]"
7
+ uvicorn data_factory.service.app:app --port 8000
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from typing import Any
13
+
14
+ __all__ = ["app", "binding", "jobs", "workspace"]
15
+
16
+
17
+ def __getattr__(name: str) -> Any:
18
+ if name == "app":
19
+ from .app import app
20
+
21
+ return app
22
+ if name in ("binding", "jobs", "workspace"):
23
+ import importlib
24
+
25
+ return importlib.import_module(f".{name}", __name__)
26
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,483 @@
1
+ """The hosted HTTP service.
2
+
3
+ A deployable API rather than the illustrative one in ``examples/``. The
4
+ difference is entirely about trust: this one never accepts a filesystem path,
5
+ sandboxes every dataset, caps uploads, expires old work, and keeps tracebacks
6
+ out of responses.
7
+
8
+ Run it::
9
+
10
+ pip install "data-factory[web]"
11
+ uvicorn data_factory.service.app:app --host 0.0.0.0 --port 8000
12
+
13
+ Configuration, all optional, all environment variables::
14
+
15
+ DATAFACTORY_WORKSPACE where datasets live (default ~/.datafactory/workspaces)
16
+ DATAFACTORY_TTL_HOURS how long an anonymous dataset survives (default 24)
17
+ DATAFACTORY_ORIGINS comma-separated CORS origins (default: same-origin only)
18
+ DATAFACTORY_API_KEY if set, every /api call must send X-API-Key
19
+ DATAFACTORY_MAX_UPLOAD_MB overrides the upload cap
20
+
21
+ The endpoints are shaped for the flow a browser actually performs: upload a
22
+ zip, look at what it is, run steps against it, download the result.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import os
28
+ import secrets
29
+ import shutil
30
+ import tempfile
31
+ import time
32
+ from contextlib import asynccontextmanager
33
+ from pathlib import Path
34
+ from typing import Any
35
+
36
+ from fastapi import Depends, FastAPI, File, Header, HTTPException, UploadFile
37
+ from fastapi.middleware.cors import CORSMiddleware
38
+ from fastapi.responses import FileResponse
39
+ from fastapi.staticfiles import StaticFiles
40
+ from starlette.concurrency import run_in_threadpool
41
+ from pydantic import BaseModel, Field
42
+
43
+ import data_factory as df
44
+
45
+ from ..core import registry
46
+ from ..core.annotations import formats
47
+ from ..core.annotations.model import AnnotationError
48
+ from ..core.registry import ParamError
49
+ from . import binding
50
+ from . import jobs as jobs_module
51
+ from .limits import BodyLimit
52
+ from .routers import annotate as annotate_router
53
+ from .routers import local as local_router
54
+ from .routers import media as media_router
55
+ from .routers import predictions as predictions_router
56
+ from .routers import promote as promote_router
57
+ from .routers import tools as tools_router
58
+ from .workspace import (
59
+ MAX_UPLOAD_BYTES,
60
+ Workspace,
61
+ WorkspaceError,
62
+ extract_zip,
63
+ make_zip,
64
+ resolve_workspace_root,
65
+ )
66
+ from ..annotate import images as images_mod
67
+ from ..annotate import projects as projects_mod
68
+ from ..annotate.store import Store, StoreError
69
+ from .deps import get_store, principal, set_jobs_manager, store_path
70
+
71
+ WORKSPACE = Workspace(
72
+ resolve_workspace_root(),
73
+ ttl_seconds=int(float(os.environ.get("DATAFACTORY_TTL_HOURS", "24")) * 3600),
74
+ )
75
+ JOBS = jobs_module.JobManager()
76
+ set_jobs_manager(JOBS)
77
+ MAX_UPLOAD = int(os.environ.get("DATAFACTORY_MAX_UPLOAD_MB", "0")) * 1024 * 1024 or MAX_UPLOAD_BYTES
78
+ API_KEY = os.environ.get("DATAFACTORY_API_KEY", "")
79
+
80
+
81
+ @asynccontextmanager
82
+ async def lifespan(app: FastAPI):
83
+ # One sweep at boot clears whatever a previous container left behind; a
84
+ # real deployment should also run it on a timer or a cron sidecar.
85
+ WORKSPACE.sweep()
86
+ yield
87
+ JOBS.shutdown()
88
+
89
+
90
+ app = FastAPI(
91
+ title="EasyAnnotate",
92
+ version=df.__version__,
93
+ summary="Dataset preparation for computer vision: convert, clean, augment, analyse.",
94
+ lifespan=lifespan,
95
+ )
96
+
97
+ # Annotation is stateful CRUD with a latency budget and cannot be expressed as
98
+ # an operation, so it gets routers of its own rather than being forced through
99
+ # the registry. See data_factory/annotate/__init__.py for the reasoning.
100
+ app.include_router(annotate_router.router)
101
+ app.include_router(media_router.router)
102
+ app.include_router(promote_router.router)
103
+ app.include_router(predictions_router.router)
104
+ app.include_router(tools_router.router)
105
+ app.include_router(local_router.router)
106
+
107
+ # Before anything reads a body. See limits.py for why the handler's own byte
108
+ # count could never do this.
109
+ app.add_middleware(BodyLimit, limit=MAX_UPLOAD)
110
+ # Outermost: a request that names another host never reaches a route.
111
+ app.add_middleware(local_router.LocalHostFence)
112
+
113
+ _origins = [o.strip() for o in os.environ.get("DATAFACTORY_ORIGINS", "").split(",") if o.strip()]
114
+ if _origins:
115
+ app.add_middleware(
116
+ CORSMiddleware,
117
+ allow_origins=_origins,
118
+ # PUT and PATCH are how the editor saves shapes and changes state.
119
+ allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE"],
120
+ allow_headers=["X-API-Key", "Content-Type"],
121
+ )
122
+
123
+
124
+ async def require_key(x_api_key: str = Header(default="")) -> None:
125
+ """Optional shared-secret gate.
126
+
127
+ Off by default so the tool runs locally with no ceremony. Set
128
+ DATAFACTORY_API_KEY before exposing it to the internet -- without it,
129
+ anyone can spend your CPU and disk.
130
+ """
131
+ if not API_KEY:
132
+ return
133
+ if not x_api_key.isascii() or not secrets.compare_digest(x_api_key, API_KEY):
134
+ raise HTTPException(401, "invalid or missing X-API-Key")
135
+
136
+
137
+ # ----------------------------------------------------------------- models
138
+
139
+ class RunRequest(BaseModel):
140
+ params: dict[str, Any] = Field(default_factory=dict)
141
+ input: str | None = Field(default=None, description="'source', or an earlier output name")
142
+ output_name: str = Field(default="", description="what to call the result")
143
+ dry_run: bool = False
144
+
145
+
146
+ # ----------------------------------------------------------------- meta
147
+
148
+ @app.get("/api/health")
149
+ def health() -> dict[str, Any]:
150
+ return {"ok": True, "version": df.__version__, "operations": len(registry.all_operations())}
151
+
152
+
153
+ @app.get("/api/operations", dependencies=[Depends(require_key)])
154
+ def list_operations() -> dict[str, Any]:
155
+ """Every operation, described so a front-end can generate its own forms.
156
+
157
+ Path parameters are omitted: the client cannot set them, so rendering an
158
+ input for one would only produce a request the service refuses.
159
+ """
160
+ return binding.public_schema()
161
+
162
+
163
+ @app.get("/api/operations/{name}", dependencies=[Depends(require_key)])
164
+ def describe_operation(name: str) -> dict[str, Any]:
165
+ try:
166
+ return binding.describe_operation(registry.get(name))
167
+ except KeyError:
168
+ raise HTTPException(404, f"unknown operation {name!r}") from None
169
+
170
+
171
+ # ----------------------------------------------------------------- datasets
172
+
173
+ @app.post("/api/datasets", dependencies=[Depends(require_key)], status_code=201)
174
+ async def upload_dataset(file: UploadFile = File(...)) -> dict[str, Any]:
175
+ """Upload a zipped dataset and get back an opaque id.
176
+
177
+ The id is the only handle a client ever holds; it never learns a path.
178
+ """
179
+ if not (file.filename or "").lower().endswith(".zip"):
180
+ raise HTTPException(415, "upload a .zip archive of your dataset")
181
+
182
+ dataset = WORKSPACE.create()
183
+ staged = Path(tempfile.mkdtemp()) / "upload.zip"
184
+ written = 0
185
+ try:
186
+ with staged.open("wb") as handle:
187
+ while chunk := await file.read(1 << 20):
188
+ written += len(chunk)
189
+ if written > MAX_UPLOAD:
190
+ raise HTTPException(
191
+ 413, f"upload exceeds the {MAX_UPLOAD / 1e6:.0f} MB limit"
192
+ )
193
+ handle.write(chunk)
194
+ # Off the event loop: unpacking a large archive is seconds of disk
195
+ # work, and on the loop it would stall every save in the editor.
196
+ stats = await run_in_threadpool(extract_zip, staged, dataset.source, workspace=WORKSPACE)
197
+ except HTTPException:
198
+ WORKSPACE.delete(dataset.id)
199
+ raise
200
+ except WorkspaceError as exc:
201
+ WORKSPACE.delete(dataset.id)
202
+ raise HTTPException(400, str(exc)) from None
203
+ finally:
204
+ shutil.rmtree(staged.parent, ignore_errors=True)
205
+
206
+ return {"id": dataset.id, "uploaded": stats, **(await run_in_threadpool(_describe, dataset))}
207
+
208
+
209
+ @app.get("/api/datasets/{dataset_id}", dependencies=[Depends(require_key)])
210
+ def dataset_info(dataset_id: str) -> dict[str, Any]:
211
+ return {"id": dataset_id, **_describe(_dataset(dataset_id))}
212
+
213
+
214
+ @app.delete("/api/datasets/{dataset_id}", dependencies=[Depends(require_key)], status_code=204)
215
+ def delete_dataset(dataset_id: str) -> None:
216
+ _dataset(dataset_id)
217
+ WORKSPACE.delete(dataset_id)
218
+
219
+
220
+ @app.get("/api/datasets/{dataset_id}/download", dependencies=[Depends(require_key)])
221
+ def download(dataset_id: str, output: str = "") -> FileResponse:
222
+ """Zip an output (or the source) for the browser to save."""
223
+ dataset = _dataset(dataset_id)
224
+ try:
225
+ source = binding.resolve_input(dataset, output or "source", WORKSPACE)
226
+ archive = make_zip(source, dataset.root / "download" / f"{output or 'source'}.zip",
227
+ workspace=WORKSPACE)
228
+ except (WorkspaceError, binding.BindingError) as exc:
229
+ raise HTTPException(404, str(exc)) from None
230
+ return FileResponse(
231
+ archive,
232
+ media_type="application/zip",
233
+ filename=f"{output or 'dataset'}-{dataset_id[:8]}.zip",
234
+ )
235
+
236
+
237
+ # ----------------------------------------------------------------- running
238
+
239
+ @app.post("/api/datasets/{dataset_id}/run/{name}", dependencies=[Depends(require_key)])
240
+ def run_operation(dataset_id: str, name: str, request: RunRequest) -> dict[str, Any]:
241
+ """Start an operation. Returns a job id immediately.
242
+
243
+ Validation happens synchronously so a bad request fails now, with the same
244
+ message the CLI would print, rather than as a failed job thirty seconds
245
+ later.
246
+ """
247
+ dataset = _dataset(dataset_id)
248
+ try:
249
+ operation = registry.get(name)
250
+ except KeyError:
251
+ raise HTTPException(404, f"unknown operation {name!r}") from None
252
+
253
+ try:
254
+ bound, output_dir = binding.bind(
255
+ operation,
256
+ request.params,
257
+ dataset=dataset,
258
+ workspace=WORKSPACE,
259
+ input_ref=request.input,
260
+ output_name=request.output_name,
261
+ )
262
+ except binding.BindingError as exc:
263
+ raise HTTPException(400, str(exc)) from None
264
+ except ParamError as exc:
265
+ raise HTTPException(422, str(exc)) from None
266
+ except WorkspaceError as exc:
267
+ raise HTTPException(400, str(exc)) from None
268
+
269
+ dry_run = request.dry_run
270
+ job_id = secrets.token_hex(8)
271
+
272
+ def work(context):
273
+ context.dry_run = dry_run
274
+ result = operation.func(bound, context)
275
+ # Provenance is recorded by Operation.run, which we bypass to keep the
276
+ # already-bound parameters, so record it explicitly here.
277
+ _record(operation.name, bound, result)
278
+ return result
279
+
280
+ # A run creates or changes an output, so the cached description is stale
281
+ # the moment it is submitted.
282
+ _invalidate(dataset_id)
283
+
284
+ job = JOBS.submit(
285
+ job_id,
286
+ operation.name,
287
+ dataset_id,
288
+ work,
289
+ output_name=output_dir.name if output_dir else "",
290
+ )
291
+ return job.as_dict()
292
+
293
+
294
+ class ExportRequest(BaseModel):
295
+ to: str = "yolo"
296
+ states: list[str] = Field(default_factory=lambda: ["reviewed"])
297
+ copy_images: bool = True
298
+ segmentation: bool = True
299
+ include_empty: bool = True
300
+
301
+
302
+ @app.post("/api/projects/{project_id}/export", dependencies=[Depends(require_key)])
303
+ def export_project(
304
+ project_id: str,
305
+ request: ExportRequest,
306
+ store: Store = Depends(get_store),
307
+ owner: str = Depends(principal),
308
+ ) -> dict[str, Any]:
309
+ """Export a project as a dataset the browser can download.
310
+
311
+ The one bridge from the annotation world back to the batch world: the
312
+ same annotate.export operation the CLI runs, as a job, into a throwaway
313
+ workspace sandbox, with the download URL handed back up front. Nothing in
314
+ the request is a path -- the database, the output folder and the image
315
+ root are all resolved here, which is what lets this exist on the hosted
316
+ service where no client may ever name a file.
317
+
318
+ Without this a hosted user could annotate and never get their work out.
319
+ """
320
+ try:
321
+ project = projects_mod.get(store, project_id, owner)
322
+ except StoreError as exc:
323
+ raise HTTPException(404, str(exc)) from None
324
+ if request.to not in formats.FORMATS:
325
+ raise HTTPException(400, f"unknown format {request.to!r}; known: {', '.join(formats.FORMATS)}")
326
+ unknown = [s for s in request.states if s not in images_mod.STATES]
327
+ if unknown or not request.states:
328
+ raise HTTPException(
329
+ 400, f"states must be some of {', '.join(images_mod.STATES)}; got {request.states!r}"
330
+ )
331
+
332
+ operation = registry.get("annotate.export")
333
+ sandbox = WORKSPACE.create()
334
+ output_dir = sandbox.output("export")
335
+ bound: dict[str, Any] = {param.name: param.default for param in operation.params}
336
+ bound.update(
337
+ {
338
+ "database": store_path(),
339
+ "project": project.id,
340
+ "output": output_dir,
341
+ "to": request.to,
342
+ "states": list(request.states),
343
+ "owner": owner,
344
+ "copy_images": request.copy_images,
345
+ "segmentation": request.segmentation,
346
+ "include_empty": request.include_empty,
347
+ }
348
+ )
349
+
350
+ job_id = secrets.token_hex(8)
351
+
352
+ def work(context):
353
+ result = operation.func(bound, context)
354
+ _record(operation.name, bound, result)
355
+ return result
356
+
357
+ job = JOBS.submit(job_id, operation.name, sandbox.id, work, output_name="export")
358
+ return {
359
+ **job.as_dict(),
360
+ "dataset": sandbox.id,
361
+ "download": f"/api/datasets/{sandbox.id}/download?output=export",
362
+ }
363
+
364
+
365
+ @app.get("/api/jobs/{job_id}", dependencies=[Depends(require_key)])
366
+ def job_status(job_id: str) -> dict[str, Any]:
367
+ job = JOBS.get(job_id)
368
+ if job is None:
369
+ raise HTTPException(404, "unknown job")
370
+ return job.as_dict()
371
+
372
+
373
+ @app.post("/api/jobs/{job_id}/cancel", dependencies=[Depends(require_key)])
374
+ def cancel_job(job_id: str) -> dict[str, Any]:
375
+ """Cooperative cancellation: the run stops at its next checkpoint."""
376
+ if not JOBS.cancel(job_id):
377
+ raise HTTPException(409, "that job has already finished")
378
+ return {"id": job_id, "state": "cancelling"}
379
+
380
+
381
+ @app.get("/api/datasets/{dataset_id}/jobs", dependencies=[Depends(require_key)])
382
+ def dataset_jobs(dataset_id: str) -> dict[str, Any]:
383
+ _dataset(dataset_id)
384
+ return {"jobs": [j.as_dict() for j in JOBS.for_dataset(dataset_id)]}
385
+
386
+
387
+ # ----------------------------------------------------------------- helpers
388
+
389
+ STATIC = Path(__file__).parent / "static"
390
+
391
+ # Only this package's own files are served here -- no user content ever lands
392
+ # in this directory, so there is nothing for a traversal to reach that is not
393
+ # already public.
394
+ app.mount("/static", StaticFiles(directory=STATIC), name="static")
395
+
396
+
397
+ @app.get("/annotate", include_in_schema=False)
398
+ def annotate_page() -> FileResponse:
399
+ """The editor.
400
+
401
+ Hand-written ES modules with no build step, so the wheel stays
402
+ self-contained and ``pip install`` keeps working offline. Phase 0 measured
403
+ a plain canvas holding 60fps with 300 shapes on a 4000x3000 image, which is
404
+ what makes that defensible rather than merely convenient.
405
+ """
406
+ return FileResponse(STATIC / "annotate.html", media_type="text/html")
407
+
408
+
409
+ @app.get("/", include_in_schema=False)
410
+ def index() -> FileResponse:
411
+ """The built-in UI.
412
+
413
+ It builds its own forms from ``/api/operations``, so an operation added to
414
+ the Python package appears in the browser with no front-end change -- the
415
+ same generation the CLI and the menu get.
416
+ """
417
+ return FileResponse(STATIC / "index.html", media_type="text/html")
418
+
419
+
420
+ def _dataset(dataset_id: str):
421
+ try:
422
+ return WORKSPACE.get(dataset_id)
423
+ except WorkspaceError as exc:
424
+ raise HTTPException(404, str(exc)) from None
425
+
426
+
427
+ #: ``_describe`` costs a full tree walk plus a complete annotation read, which
428
+ #: is O(dataset) per request. That is affordable for a page load and hopeless
429
+ #: for anything that polls, so the answer is held briefly and dropped whenever
430
+ #: a run could have changed it.
431
+ _DESCRIBE_TTL = 10.0
432
+ _describe_cache: dict[str, tuple[float, dict[str, Any]]] = {}
433
+
434
+
435
+ def _invalidate(dataset_id: str) -> None:
436
+ _describe_cache.pop(dataset_id, None)
437
+
438
+
439
+ def _describe(dataset) -> dict[str, Any]:
440
+ """What the dataset is, as far as we can tell, for the UI to show."""
441
+ cached = _describe_cache.get(dataset.id)
442
+ now = time.time()
443
+ if cached is not None and now - cached[0] < _DESCRIBE_TTL:
444
+ # Recompute the countdown; everything else is stable for the window.
445
+ payload = dict(cached[1])
446
+ payload["expires_in_seconds"] = max(
447
+ 0, int(dataset.created_at() + WORKSPACE.ttl_seconds - now)
448
+ )
449
+ return payload
450
+ payload = _build_description(dataset)
451
+ _describe_cache[dataset.id] = (now, payload)
452
+ return dict(payload)
453
+
454
+
455
+ def _build_description(dataset) -> dict[str, Any]:
456
+ payload: dict[str, Any] = {
457
+ "inputs": binding.available_inputs(dataset),
458
+ "size_bytes": dataset.size_bytes(),
459
+ "expires_in_seconds": max(
460
+ 0, int(dataset.created_at() + WORKSPACE.ttl_seconds - time.time())
461
+ ),
462
+ }
463
+ try:
464
+ loaded = formats.read_dataset(dataset.source, "auto", recursive=True)
465
+ payload["format"] = loaded.source_format
466
+ payload["stats"] = loaded.stats()
467
+ payload["classes"] = loaded.labels
468
+ payload["warnings"] = loaded.warnings[:10]
469
+ except AnnotationError as exc:
470
+ # Not fatal: plenty of useful operations work on a folder of images
471
+ # with no annotations at all.
472
+ payload["format"] = None
473
+ payload["detection_error"] = str(exc)
474
+ return payload
475
+
476
+
477
+ def _record(name: str, bound: dict[str, Any], result) -> None:
478
+ try:
479
+ from ..core import provenance
480
+
481
+ provenance.record(name, bound, result, version=df.__version__)
482
+ except Exception:
483
+ pass