easyannotate 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_factory/__init__.py +82 -0
- data_factory/__main__.py +6 -0
- data_factory/annotate/__init__.py +15 -0
- data_factory/annotate/attach.py +398 -0
- data_factory/annotate/demo.py +161 -0
- data_factory/annotate/images.py +443 -0
- data_factory/annotate/labels.py +399 -0
- data_factory/annotate/loop.py +359 -0
- data_factory/annotate/projects.py +207 -0
- data_factory/annotate/review.py +471 -0
- data_factory/annotate/schema.sql +171 -0
- data_factory/annotate/shapes.py +488 -0
- data_factory/annotate/store.py +292 -0
- data_factory/annotate/tools/__init__.py +39 -0
- data_factory/annotate/tools/fake.py +170 -0
- data_factory/annotate/tools/install.py +254 -0
- data_factory/annotate/tools/local.py +389 -0
- data_factory/annotate/tools/postprocess.py +175 -0
- data_factory/annotate/tools/protocol.py +238 -0
- data_factory/annotate/tools/registry.py +196 -0
- data_factory/annotate/tools/results.py +83 -0
- data_factory/annotate/tools/serve.py +106 -0
- data_factory/cli/__init__.py +21 -0
- data_factory/cli/main.py +837 -0
- data_factory/cli/menu.py +173 -0
- data_factory/cli/render.py +158 -0
- data_factory/core/__init__.py +11 -0
- data_factory/core/annotations/__init__.py +5 -0
- data_factory/core/annotations/coco.py +394 -0
- data_factory/core/annotations/csvfmt.py +174 -0
- data_factory/core/annotations/dota.py +98 -0
- data_factory/core/annotations/formats.py +626 -0
- data_factory/core/annotations/labelme.py +153 -0
- data_factory/core/annotations/model.py +334 -0
- data_factory/core/annotations/voc.py +231 -0
- data_factory/core/annotations/yolo.py +283 -0
- data_factory/core/bktree.py +89 -0
- data_factory/core/fs.py +245 -0
- data_factory/core/imaging.py +626 -0
- data_factory/core/ops/__init__.py +44 -0
- data_factory/core/ops/analyze.py +383 -0
- data_factory/core/ops/annotate_export.py +326 -0
- data_factory/core/ops/audit.py +274 -0
- data_factory/core/ops/augment.py +454 -0
- data_factory/core/ops/collect.py +241 -0
- data_factory/core/ops/combine.py +582 -0
- data_factory/core/ops/convert.py +212 -0
- data_factory/core/ops/dataset.py +246 -0
- data_factory/core/ops/dedup.py +235 -0
- data_factory/core/ops/export.py +168 -0
- data_factory/core/ops/image.py +407 -0
- data_factory/core/ops/labels.py +215 -0
- data_factory/core/ops/pairing.py +133 -0
- data_factory/core/ops/rename.py +154 -0
- data_factory/core/ops/report.py +323 -0
- data_factory/core/ops/visualize.py +164 -0
- data_factory/core/pairs.py +144 -0
- data_factory/core/parallel.py +191 -0
- data_factory/core/provenance.py +136 -0
- data_factory/core/registry.py +339 -0
- data_factory/core/stats.py +403 -0
- data_factory/core/types.py +148 -0
- data_factory/core/writer.py +194 -0
- data_factory/desktop.py +207 -0
- data_factory/hardware.py +243 -0
- data_factory/pipeline.py +192 -0
- data_factory/py.typed +0 -0
- data_factory/service/__init__.py +26 -0
- data_factory/service/app.py +483 -0
- data_factory/service/binding.py +174 -0
- data_factory/service/deps.py +133 -0
- data_factory/service/jobs.py +177 -0
- data_factory/service/limits.py +71 -0
- data_factory/service/routers/__init__.py +11 -0
- data_factory/service/routers/annotate.py +582 -0
- data_factory/service/routers/local.py +464 -0
- data_factory/service/routers/media.py +154 -0
- data_factory/service/routers/predictions.py +116 -0
- data_factory/service/routers/promote.py +182 -0
- data_factory/service/routers/tools.py +308 -0
- data_factory/service/static/annotate/api.js +246 -0
- data_factory/service/static/annotate/app.js +1696 -0
- data_factory/service/static/annotate/assist.js +106 -0
- data_factory/service/static/annotate/filmstrip.js +286 -0
- data_factory/service/static/annotate/hit.js +146 -0
- data_factory/service/static/annotate/labels.js +197 -0
- data_factory/service/static/annotate/render.js +286 -0
- data_factory/service/static/annotate/scene.js +277 -0
- data_factory/service/static/annotate/shapelist.js +72 -0
- data_factory/service/static/annotate/tools/box.js +96 -0
- data_factory/service/static/annotate/tools/line.js +96 -0
- data_factory/service/static/annotate/tools/obb.js +88 -0
- data_factory/service/static/annotate/tools/point.js +49 -0
- data_factory/service/static/annotate/tools/polygon.js +110 -0
- data_factory/service/static/annotate/tools/select.js +227 -0
- data_factory/service/static/annotate/undo.js +176 -0
- data_factory/service/static/annotate/viewport.js +156 -0
- data_factory/service/static/annotate.html +390 -0
- data_factory/service/static/index.html +632 -0
- data_factory/service/workspace.py +244 -0
- easyannotate/__init__.py +64 -0
- easyannotate/__main__.py +5 -0
- easyannotate/project.py +181 -0
- easyannotate/py.typed +0 -0
- easyannotate-0.4.0.dist-info/METADATA +767 -0
- easyannotate-0.4.0.dist-info/RECORD +109 -0
- easyannotate-0.4.0.dist-info/WHEEL +4 -0
- easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
- easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
"""Turning a web request into operation parameters, without trusting paths.
|
|
2
|
+
|
|
3
|
+
Operations take real filesystem paths. A browser must never supply one, so this
|
|
4
|
+
module is the translation layer: the client picks *what* to run on by logical
|
|
5
|
+
name -- ``"source"``, or the name of an earlier output -- and every actual path
|
|
6
|
+
is derived from the dataset sandbox here.
|
|
7
|
+
|
|
8
|
+
The rule enforced below is blunt on purpose: **any path-kind parameter sent by
|
|
9
|
+
a client is rejected outright**, rather than sanitised. Sanitising invites an
|
|
10
|
+
arms race over encodings, symlinks and UNC paths; refusing does not.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from ..core.registry import Operation, ParamError
|
|
19
|
+
from .workspace import Dataset, Workspace, safe_segment
|
|
20
|
+
|
|
21
|
+
#: Parameter kinds that name a location on disk. None may come from a client.
|
|
22
|
+
PATH_KINDS = frozenset({"dir", "file", "path", "dirs"})
|
|
23
|
+
|
|
24
|
+
#: Parameters that name where an operation writes, rather than what it reads.
|
|
25
|
+
OUTPUT_NAMES = frozenset({"output", "quarantine"})
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class BindingError(Exception):
|
|
29
|
+
"""Raised when a request cannot be turned into a safe parameter set."""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def resolve_input(dataset: Dataset, reference: str | None, workspace: Workspace) -> Path:
|
|
33
|
+
"""Map a logical input name onto a real directory inside the sandbox.
|
|
34
|
+
|
|
35
|
+
``None`` or ``"source"`` means the uploaded dataset; anything else names an
|
|
36
|
+
output produced by an earlier step, which is what makes chaining possible
|
|
37
|
+
without ever exposing a path.
|
|
38
|
+
"""
|
|
39
|
+
if reference in (None, "", "source"):
|
|
40
|
+
return workspace.assert_inside(dataset.source)
|
|
41
|
+
candidate = dataset.output(reference)
|
|
42
|
+
resolved = workspace.assert_inside(candidate)
|
|
43
|
+
if not resolved.is_dir():
|
|
44
|
+
raise BindingError(
|
|
45
|
+
f"no step has produced {reference!r} yet. Available: "
|
|
46
|
+
f"{', '.join(available_inputs(dataset)) or 'source'}"
|
|
47
|
+
)
|
|
48
|
+
return resolved
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def available_inputs(dataset: Dataset) -> list[str]:
|
|
52
|
+
"""Everything that can currently be used as an input, for the UI to offer."""
|
|
53
|
+
names = ["source"] if any(dataset.source.iterdir()) else []
|
|
54
|
+
if dataset.outputs.is_dir():
|
|
55
|
+
names.extend(sorted(p.name for p in dataset.outputs.iterdir() if p.is_dir()))
|
|
56
|
+
return names
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def bind(
|
|
60
|
+
operation: Operation,
|
|
61
|
+
supplied: dict[str, Any],
|
|
62
|
+
*,
|
|
63
|
+
dataset: Dataset,
|
|
64
|
+
workspace: Workspace,
|
|
65
|
+
input_ref: str | None = None,
|
|
66
|
+
output_name: str = "",
|
|
67
|
+
) -> tuple[dict[str, Any], Path | None]:
|
|
68
|
+
"""Build a safe, fully-populated parameter set for one run.
|
|
69
|
+
|
|
70
|
+
Returns the parameters and the output directory, if the operation writes
|
|
71
|
+
one. Raises :class:`BindingError` for anything a client should not be able
|
|
72
|
+
to ask for, and lets :class:`ParamError` through for ordinary validation --
|
|
73
|
+
so the web API reports bad values exactly as the CLI does.
|
|
74
|
+
"""
|
|
75
|
+
offered = dict(supplied or {})
|
|
76
|
+
|
|
77
|
+
# Refuse rather than sanitise. A rejected request is a bug report; a
|
|
78
|
+
# silently rewritten one is a vulnerability waiting for an encoding trick.
|
|
79
|
+
for spec in operation.params:
|
|
80
|
+
if spec.kind in PATH_KINDS and spec.name in offered:
|
|
81
|
+
raise BindingError(
|
|
82
|
+
f"{spec.name!r} is a path and cannot be set directly. Choose an "
|
|
83
|
+
f"input with 'input', and name the result with 'output_name'."
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
output_dir: Path | None = None
|
|
87
|
+
for spec in operation.params:
|
|
88
|
+
if spec.kind not in PATH_KINDS:
|
|
89
|
+
continue
|
|
90
|
+
|
|
91
|
+
if spec.name in OUTPUT_NAMES:
|
|
92
|
+
target = dataset.output(output_name or f"{operation.name.replace('.', '_')}")
|
|
93
|
+
output_dir = workspace.assert_inside(target)
|
|
94
|
+
if spec.kind == "file":
|
|
95
|
+
# report.html writes a file, not a directory.
|
|
96
|
+
output_dir.mkdir(parents=True, exist_ok=True)
|
|
97
|
+
offered[spec.name] = str(output_dir / _filename_for(operation))
|
|
98
|
+
else:
|
|
99
|
+
offered[spec.name] = str(output_dir)
|
|
100
|
+
continue
|
|
101
|
+
|
|
102
|
+
source = resolve_input(dataset, input_ref, workspace)
|
|
103
|
+
if spec.kind == "dirs":
|
|
104
|
+
offered[spec.name] = [str(source)]
|
|
105
|
+
elif spec.kind == "file":
|
|
106
|
+
offered[spec.name] = str(_find_file(source, operation))
|
|
107
|
+
else:
|
|
108
|
+
offered[spec.name] = str(source)
|
|
109
|
+
|
|
110
|
+
bound = operation.bind(offered)
|
|
111
|
+
return bound, output_dir
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _filename_for(operation: Operation) -> str:
|
|
115
|
+
return "report.html" if operation.name == "report.html" else "result.json"
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _find_file(folder: Path, operation: Operation) -> Path:
|
|
119
|
+
"""Locate the single file a file-kind input wants.
|
|
120
|
+
|
|
121
|
+
Only ``inspect.coco`` needs this, and it wants an instances.json; guessing
|
|
122
|
+
is acceptable because the alternative is asking a browser for a path.
|
|
123
|
+
"""
|
|
124
|
+
for pattern in ("**/instances*.json", "**/_annotations.coco.json", "**/*.json"):
|
|
125
|
+
for found in sorted(folder.glob(pattern)):
|
|
126
|
+
if found.is_file():
|
|
127
|
+
return found
|
|
128
|
+
raise BindingError(f"{operation.name} needs a json file and none was found in the dataset")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def describe_operation(operation: Operation) -> dict[str, Any]:
|
|
132
|
+
"""The operation as a web form should render it.
|
|
133
|
+
|
|
134
|
+
Path parameters are stripped: the client cannot set them, so showing them
|
|
135
|
+
would only invite a request the service will refuse.
|
|
136
|
+
"""
|
|
137
|
+
payload = operation.as_dict()
|
|
138
|
+
payload["params"] = [p for p in payload["params"] if p["kind"] not in PATH_KINDS]
|
|
139
|
+
payload["writes_output"] = any(
|
|
140
|
+
p.name in OUTPUT_NAMES for p in operation.params
|
|
141
|
+
)
|
|
142
|
+
return payload
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def public_schema() -> dict[str, Any]:
|
|
146
|
+
"""Every operation a browser can usefully render a form for.
|
|
147
|
+
|
|
148
|
+
The ``hidden_from_web`` ones are filtered here and only here -- they stay
|
|
149
|
+
in ``datafactory list``, in ``schema``, and in recipes. An operation whose
|
|
150
|
+
real input is an annotation project rather than a folder has nothing the
|
|
151
|
+
batch panel could ask for, so offering it would only produce a refusal.
|
|
152
|
+
"""
|
|
153
|
+
from ..core import registry
|
|
154
|
+
|
|
155
|
+
return {
|
|
156
|
+
"version": 1,
|
|
157
|
+
"operations": [
|
|
158
|
+
describe_operation(op)
|
|
159
|
+
for op in registry.all_operations()
|
|
160
|
+
if not op.hidden_from_web
|
|
161
|
+
],
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
__all__ = [
|
|
166
|
+
"BindingError",
|
|
167
|
+
"ParamError",
|
|
168
|
+
"available_inputs",
|
|
169
|
+
"bind",
|
|
170
|
+
"describe_operation",
|
|
171
|
+
"public_schema",
|
|
172
|
+
"resolve_input",
|
|
173
|
+
"safe_segment",
|
|
174
|
+
]
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Shared request dependencies: the key gate, the store, and where files live.
|
|
2
|
+
|
|
3
|
+
Split out of ``app.py`` so the annotation routers can depend on the same gate
|
|
4
|
+
without importing the application module they are mounted into.
|
|
5
|
+
|
|
6
|
+
**Layout.** The dataset sandboxes live under the workspace root and are
|
|
7
|
+
transient -- swept on TTL, zipped for download, addressed by an opaque id.
|
|
8
|
+
Annotation projects are not transient, so nothing of theirs is stored in
|
|
9
|
+
there::
|
|
10
|
+
|
|
11
|
+
~/.datafactory/
|
|
12
|
+
workspaces/ <- dataset sandboxes, swept, zipped, served
|
|
13
|
+
annotations.db <- the project database
|
|
14
|
+
projects/<id>/ <- project media
|
|
15
|
+
|
|
16
|
+
A database file inside a sandbox would eventually be swept out from under a
|
|
17
|
+
user, packed into somebody's download, or served by a static route. Putting it
|
|
18
|
+
one level up costs nothing and removes all three.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import os
|
|
24
|
+
import secrets
|
|
25
|
+
import threading
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
from fastapi import Header, HTTPException
|
|
30
|
+
|
|
31
|
+
from ..annotate.store import LOCAL_OWNER, Store
|
|
32
|
+
from .workspace import resolve_workspace_root
|
|
33
|
+
|
|
34
|
+
#: Read per call rather than captured at import, so a test (or a container
|
|
35
|
+
#: restart with new settings) does not need this module reloaded to see it.
|
|
36
|
+
def api_key() -> str:
|
|
37
|
+
return os.environ.get("DATAFACTORY_API_KEY", "")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
async def require_key(x_api_key: str = Header(default="")) -> None:
|
|
41
|
+
"""Optional shared-secret gate.
|
|
42
|
+
|
|
43
|
+
Off by default so the tool runs locally with no ceremony. Set
|
|
44
|
+
``DATAFACTORY_API_KEY`` before exposing it to the internet -- without it,
|
|
45
|
+
anyone can spend your CPU and disk.
|
|
46
|
+
"""
|
|
47
|
+
key = api_key()
|
|
48
|
+
if not key:
|
|
49
|
+
return
|
|
50
|
+
# isascii first: compare_digest raises on non-ASCII text -- a 500, not a 401.
|
|
51
|
+
if not x_api_key.isascii() or not secrets.compare_digest(x_api_key, key):
|
|
52
|
+
raise HTTPException(401, "invalid or missing X-API-Key")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
_jobs: Any = None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def set_jobs_manager(manager: Any) -> None:
|
|
59
|
+
"""The app registers its JobManager here so routers can submit work
|
|
60
|
+
without importing the app module (which imports the routers)."""
|
|
61
|
+
global _jobs
|
|
62
|
+
_jobs = manager
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def jobs_manager() -> Any:
|
|
66
|
+
return _jobs
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def data_root() -> Path:
|
|
70
|
+
"""The directory holding the database and project media."""
|
|
71
|
+
return resolve_workspace_root().parent
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def store_path() -> Path:
|
|
75
|
+
return data_root() / "annotations.db"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
_stores: dict[Path, Store] = {}
|
|
79
|
+
_stores_lock = threading.Lock()
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def get_store() -> Store:
|
|
83
|
+
"""The annotation store for the configured workspace.
|
|
84
|
+
|
|
85
|
+
Cached *by path* rather than as a single global. Tests point
|
|
86
|
+
``DATAFACTORY_WORKSPACE`` somewhere new between cases, and a plain global
|
|
87
|
+
would keep serving the first one they happened to create.
|
|
88
|
+
"""
|
|
89
|
+
path = store_path()
|
|
90
|
+
with _stores_lock:
|
|
91
|
+
existing = _stores.get(path)
|
|
92
|
+
if existing is None:
|
|
93
|
+
existing = Store(path)
|
|
94
|
+
_stores[path] = existing
|
|
95
|
+
return existing
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def reset_stores() -> None:
|
|
99
|
+
"""Drop cached connections. For tests and for a clean shutdown."""
|
|
100
|
+
with _stores_lock:
|
|
101
|
+
for store in _stores.values():
|
|
102
|
+
store.close()
|
|
103
|
+
_stores.clear()
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
async def principal() -> str:
|
|
107
|
+
"""Who is asking.
|
|
108
|
+
|
|
109
|
+
Locally this is always the same answer, and that is the point: every query
|
|
110
|
+
below is already scoped by owner, so Phase 5 replaces this one function
|
|
111
|
+
with a real session lookup instead of revisiting every route.
|
|
112
|
+
"""
|
|
113
|
+
return LOCAL_OWNER
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def media_root(project_id: str) -> Path:
|
|
117
|
+
"""Where a managed project's images are kept."""
|
|
118
|
+
return data_root() / "projects" / project_id
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def resolve_media(root: Path, rel_path: str) -> Path:
|
|
122
|
+
"""Join a stored relative path onto a root, refusing anything that escapes.
|
|
123
|
+
|
|
124
|
+
The client never sends this path -- it names an image id and the path comes
|
|
125
|
+
out of the database. The check is here anyway: a row could have been
|
|
126
|
+
written by an older build, and a symlink planted under the root would
|
|
127
|
+
otherwise redirect the read.
|
|
128
|
+
"""
|
|
129
|
+
base = root.resolve()
|
|
130
|
+
target = (base / rel_path).resolve()
|
|
131
|
+
if target != base and base not in target.parents:
|
|
132
|
+
raise HTTPException(400, "that image is outside the project")
|
|
133
|
+
return target
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"""Background jobs with progress and cancellation.
|
|
2
|
+
|
|
3
|
+
Dataset work outlives an HTTP request -- augmenting 3,000 images takes minutes,
|
|
4
|
+
and a browser will not hold a connection open for that. So a run returns a job
|
|
5
|
+
id immediately and the client polls.
|
|
6
|
+
|
|
7
|
+
This is a single-process, in-memory manager, which is the right size for one
|
|
8
|
+
container serving a modest site. It is deliberately shaped so that swapping in
|
|
9
|
+
Redis or Celery later means replacing this file and nothing else: the rest of
|
|
10
|
+
the service only ever calls ``submit``, ``get`` and ``cancel``.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
import traceback
|
|
18
|
+
from collections import OrderedDict
|
|
19
|
+
from collections.abc import Callable
|
|
20
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
21
|
+
from dataclasses import dataclass, field
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
from ..core.types import Cancelled, Context, OpResult
|
|
26
|
+
|
|
27
|
+
#: Runs at once. Dataset work is CPU-bound and already multi-threaded inside
|
|
28
|
+
#: each operation, so letting several run concurrently mostly adds contention.
|
|
29
|
+
MAX_CONCURRENT = 2
|
|
30
|
+
|
|
31
|
+
#: Finished jobs kept for polling before being evicted.
|
|
32
|
+
MAX_REMEMBERED = 500
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class Job:
|
|
37
|
+
id: str
|
|
38
|
+
operation: str
|
|
39
|
+
dataset_id: str
|
|
40
|
+
state: str = "queued" # queued | running | finished | failed | cancelled
|
|
41
|
+
done: int = 0
|
|
42
|
+
total: int = 0
|
|
43
|
+
message: str = ""
|
|
44
|
+
created_at: float = field(default_factory=time.time)
|
|
45
|
+
started_at: float = 0.0
|
|
46
|
+
finished_at: float = 0.0
|
|
47
|
+
result: dict[str, Any] | None = None
|
|
48
|
+
error: str = ""
|
|
49
|
+
output_name: str = ""
|
|
50
|
+
|
|
51
|
+
def as_dict(self) -> dict[str, Any]:
|
|
52
|
+
elapsed = (self.finished_at or time.time()) - (self.started_at or self.created_at)
|
|
53
|
+
return {
|
|
54
|
+
"id": self.id,
|
|
55
|
+
"operation": self.operation,
|
|
56
|
+
"dataset": self.dataset_id,
|
|
57
|
+
"state": self.state,
|
|
58
|
+
"progress": {
|
|
59
|
+
"done": self.done,
|
|
60
|
+
"total": self.total,
|
|
61
|
+
"message": self.message,
|
|
62
|
+
# A percentage the UI can render without dividing by zero.
|
|
63
|
+
"percent": round(100 * self.done / self.total, 1) if self.total else None,
|
|
64
|
+
},
|
|
65
|
+
"elapsed_seconds": round(elapsed, 1),
|
|
66
|
+
"output": self.output_name or None,
|
|
67
|
+
"result": self.result,
|
|
68
|
+
"error": self.error or None,
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class JobManager:
|
|
73
|
+
"""Runs operations off the request thread, with progress and cancellation."""
|
|
74
|
+
|
|
75
|
+
def __init__(self, max_concurrent: int = MAX_CONCURRENT) -> None:
|
|
76
|
+
self._pool = ThreadPoolExecutor(max_workers=max_concurrent, thread_name_prefix="dfx-job")
|
|
77
|
+
self._lock = threading.Lock()
|
|
78
|
+
self._jobs: OrderedDict[str, Job] = OrderedDict()
|
|
79
|
+
self._cancelled: set[str] = set()
|
|
80
|
+
|
|
81
|
+
# -- api ---------------------------------------------------------------
|
|
82
|
+
def submit(
|
|
83
|
+
self,
|
|
84
|
+
job_id: str,
|
|
85
|
+
operation: str,
|
|
86
|
+
dataset_id: str,
|
|
87
|
+
work: Callable[[Context], OpResult],
|
|
88
|
+
*,
|
|
89
|
+
output_name: str = "",
|
|
90
|
+
) -> Job:
|
|
91
|
+
job = Job(id=job_id, operation=operation, dataset_id=dataset_id, output_name=output_name)
|
|
92
|
+
with self._lock:
|
|
93
|
+
self._jobs[job_id] = job
|
|
94
|
+
self._evict_locked()
|
|
95
|
+
self._pool.submit(self._run, job, work)
|
|
96
|
+
return job
|
|
97
|
+
|
|
98
|
+
def get(self, job_id: str) -> Job | None:
|
|
99
|
+
with self._lock:
|
|
100
|
+
return self._jobs.get(job_id)
|
|
101
|
+
|
|
102
|
+
def cancel(self, job_id: str) -> bool:
|
|
103
|
+
with self._lock:
|
|
104
|
+
job = self._jobs.get(job_id)
|
|
105
|
+
if job is None or job.state in ("finished", "failed", "cancelled"):
|
|
106
|
+
return False
|
|
107
|
+
self._cancelled.add(job_id)
|
|
108
|
+
if job.state == "queued":
|
|
109
|
+
# Never started, so nothing will poll the cancel flag for us.
|
|
110
|
+
job.state = "cancelled"
|
|
111
|
+
job.finished_at = time.time()
|
|
112
|
+
return True
|
|
113
|
+
|
|
114
|
+
def for_dataset(self, dataset_id: str) -> list[Job]:
|
|
115
|
+
with self._lock:
|
|
116
|
+
return [j for j in self._jobs.values() if j.dataset_id == dataset_id]
|
|
117
|
+
|
|
118
|
+
# -- internals ---------------------------------------------------------
|
|
119
|
+
def _run(self, job: Job, work: Callable[[Context], OpResult]) -> None:
|
|
120
|
+
with self._lock:
|
|
121
|
+
if job.id in self._cancelled:
|
|
122
|
+
job.state = "cancelled"
|
|
123
|
+
job.finished_at = time.time()
|
|
124
|
+
return
|
|
125
|
+
job.state = "running"
|
|
126
|
+
job.started_at = time.time()
|
|
127
|
+
|
|
128
|
+
def progress(done: int, total: int, message: str = "") -> None:
|
|
129
|
+
with self._lock:
|
|
130
|
+
job.done, job.total, job.message = done, total, message
|
|
131
|
+
|
|
132
|
+
def cancelled() -> bool:
|
|
133
|
+
with self._lock:
|
|
134
|
+
return job.id in self._cancelled
|
|
135
|
+
|
|
136
|
+
context = Context(progress=progress, cancel_check=cancelled)
|
|
137
|
+
try:
|
|
138
|
+
result = work(context)
|
|
139
|
+
with self._lock:
|
|
140
|
+
# Actions are the per-file audit trail: useful in a terminal,
|
|
141
|
+
# far too large to ship to a browser for a 10,000-file run.
|
|
142
|
+
job.result = result.as_dict(include_actions=False)
|
|
143
|
+
job.state = "finished" if result.ok else "failed"
|
|
144
|
+
if not result.ok:
|
|
145
|
+
job.error = "; ".join(result.errors) or "the operation reported failure"
|
|
146
|
+
job.finished_at = time.time()
|
|
147
|
+
except Cancelled:
|
|
148
|
+
with self._lock:
|
|
149
|
+
job.state = "cancelled"
|
|
150
|
+
job.finished_at = time.time()
|
|
151
|
+
except Exception as exc:
|
|
152
|
+
# The traceback goes to the log, not to the client: it can carry
|
|
153
|
+
# host paths and package layout that a public API should not leak.
|
|
154
|
+
traceback.print_exc()
|
|
155
|
+
with self._lock:
|
|
156
|
+
job.state = "failed"
|
|
157
|
+
job.error = f"{type(exc).__name__}: {exc}"
|
|
158
|
+
job.finished_at = time.time()
|
|
159
|
+
finally:
|
|
160
|
+
with self._lock:
|
|
161
|
+
self._cancelled.discard(job.id)
|
|
162
|
+
|
|
163
|
+
def _evict_locked(self) -> None:
|
|
164
|
+
while len(self._jobs) > MAX_REMEMBERED:
|
|
165
|
+
for key, job in self._jobs.items():
|
|
166
|
+
if job.state in ("finished", "failed", "cancelled"):
|
|
167
|
+
del self._jobs[key]
|
|
168
|
+
break
|
|
169
|
+
else:
|
|
170
|
+
return
|
|
171
|
+
|
|
172
|
+
def shutdown(self) -> None:
|
|
173
|
+
self._pool.shutdown(wait=False, cancel_futures=True)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def output_exists(path: Path | None) -> bool:
|
|
177
|
+
return bool(path and path.is_dir() and any(path.iterdir()))
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Request-size limits that act before a body is read.
|
|
2
|
+
|
|
3
|
+
A handler that counts bytes as it reads an ``UploadFile`` caps nothing:
|
|
4
|
+
Starlette has already parsed the multipart body -- spooling it to a temporary
|
|
5
|
+
file on disk -- before the handler runs. A 100 GB POST fills the disk first and
|
|
6
|
+
is refused afterwards. So the limit lives here, in front of the app, where the
|
|
7
|
+
only thing read is the header.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
#: Room for multipart framing around a file that is itself within the limit.
|
|
16
|
+
FRAMING_SLACK = 1024 * 1024
|
|
17
|
+
|
|
18
|
+
_BODY_METHODS = {b"POST", b"PUT", b"PATCH"}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class BodyLimit:
|
|
22
|
+
"""Refuse a request whose declared size exceeds ``limit`` -- before reading it.
|
|
23
|
+
|
|
24
|
+
A body with no ``Content-Length`` (chunked) is refused too, with 411:
|
|
25
|
+
without a declared length the only way to enforce a limit is to read the
|
|
26
|
+
body, which is the problem this exists to avoid. Browsers, httpx and curl
|
|
27
|
+
all declare the length of a file upload.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
def __init__(self, app: Any, limit: int) -> None:
|
|
31
|
+
self.app = app
|
|
32
|
+
self.limit = int(limit) + FRAMING_SLACK
|
|
33
|
+
|
|
34
|
+
async def __call__(self, scope: dict, receive: Any, send: Any) -> None:
|
|
35
|
+
if scope["type"] != "http" or scope["method"].encode() not in _BODY_METHODS:
|
|
36
|
+
await self.app(scope, receive, send)
|
|
37
|
+
return
|
|
38
|
+
|
|
39
|
+
headers = dict(scope.get("headers") or [])
|
|
40
|
+
declared = headers.get(b"content-length")
|
|
41
|
+
if declared is None:
|
|
42
|
+
if headers.get(b"transfer-encoding", b"").lower() == b"chunked":
|
|
43
|
+
await _refuse(send, 411, "send a Content-Length with uploads")
|
|
44
|
+
return
|
|
45
|
+
elif not declared.isdigit():
|
|
46
|
+
await _refuse(send, 400, "Content-Length is not a number")
|
|
47
|
+
return
|
|
48
|
+
elif int(declared) > self.limit:
|
|
49
|
+
mb = (self.limit - FRAMING_SLACK) / 1e6
|
|
50
|
+
await _refuse(send, 413, f"upload exceeds the {mb:.0f} MB limit")
|
|
51
|
+
return
|
|
52
|
+
|
|
53
|
+
await self.app(scope, receive, send)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
async def _refuse(send: Any, status: int, detail: str) -> None:
|
|
57
|
+
body = json.dumps({"detail": detail}).encode()
|
|
58
|
+
await send(
|
|
59
|
+
{
|
|
60
|
+
"type": "http.response.start",
|
|
61
|
+
"status": status,
|
|
62
|
+
"headers": [
|
|
63
|
+
(b"content-type", b"application/json"),
|
|
64
|
+
(b"content-length", str(len(body)).encode()),
|
|
65
|
+
# The client may still be sending; tell it not to reuse a
|
|
66
|
+
# connection with an unread body on it.
|
|
67
|
+
(b"connection", b"close"),
|
|
68
|
+
],
|
|
69
|
+
}
|
|
70
|
+
)
|
|
71
|
+
await send({"type": "http.response.body", "body": body})
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""HTTP routers.
|
|
2
|
+
|
|
3
|
+
The batch API in ``app.py`` is request/response over folders. Annotation is
|
|
4
|
+
stateful CRUD with a latency budget, so it lives here instead of being forced
|
|
5
|
+
through the operation registry -- see ``data_factory/annotate/__init__.py`` for
|
|
6
|
+
why that boundary exists.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
__all__ = ["annotate", "media", "predictions", "promote", "tools"]
|