easyannotate 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. data_factory/__init__.py +82 -0
  2. data_factory/__main__.py +6 -0
  3. data_factory/annotate/__init__.py +15 -0
  4. data_factory/annotate/attach.py +398 -0
  5. data_factory/annotate/demo.py +161 -0
  6. data_factory/annotate/images.py +443 -0
  7. data_factory/annotate/labels.py +399 -0
  8. data_factory/annotate/loop.py +359 -0
  9. data_factory/annotate/projects.py +207 -0
  10. data_factory/annotate/review.py +471 -0
  11. data_factory/annotate/schema.sql +171 -0
  12. data_factory/annotate/shapes.py +488 -0
  13. data_factory/annotate/store.py +292 -0
  14. data_factory/annotate/tools/__init__.py +39 -0
  15. data_factory/annotate/tools/fake.py +170 -0
  16. data_factory/annotate/tools/install.py +254 -0
  17. data_factory/annotate/tools/local.py +389 -0
  18. data_factory/annotate/tools/postprocess.py +175 -0
  19. data_factory/annotate/tools/protocol.py +238 -0
  20. data_factory/annotate/tools/registry.py +196 -0
  21. data_factory/annotate/tools/results.py +83 -0
  22. data_factory/annotate/tools/serve.py +106 -0
  23. data_factory/cli/__init__.py +21 -0
  24. data_factory/cli/main.py +837 -0
  25. data_factory/cli/menu.py +173 -0
  26. data_factory/cli/render.py +158 -0
  27. data_factory/core/__init__.py +11 -0
  28. data_factory/core/annotations/__init__.py +5 -0
  29. data_factory/core/annotations/coco.py +394 -0
  30. data_factory/core/annotations/csvfmt.py +174 -0
  31. data_factory/core/annotations/dota.py +98 -0
  32. data_factory/core/annotations/formats.py +626 -0
  33. data_factory/core/annotations/labelme.py +153 -0
  34. data_factory/core/annotations/model.py +334 -0
  35. data_factory/core/annotations/voc.py +231 -0
  36. data_factory/core/annotations/yolo.py +283 -0
  37. data_factory/core/bktree.py +89 -0
  38. data_factory/core/fs.py +245 -0
  39. data_factory/core/imaging.py +626 -0
  40. data_factory/core/ops/__init__.py +44 -0
  41. data_factory/core/ops/analyze.py +383 -0
  42. data_factory/core/ops/annotate_export.py +326 -0
  43. data_factory/core/ops/audit.py +274 -0
  44. data_factory/core/ops/augment.py +454 -0
  45. data_factory/core/ops/collect.py +241 -0
  46. data_factory/core/ops/combine.py +582 -0
  47. data_factory/core/ops/convert.py +212 -0
  48. data_factory/core/ops/dataset.py +246 -0
  49. data_factory/core/ops/dedup.py +235 -0
  50. data_factory/core/ops/export.py +168 -0
  51. data_factory/core/ops/image.py +407 -0
  52. data_factory/core/ops/labels.py +215 -0
  53. data_factory/core/ops/pairing.py +133 -0
  54. data_factory/core/ops/rename.py +154 -0
  55. data_factory/core/ops/report.py +323 -0
  56. data_factory/core/ops/visualize.py +164 -0
  57. data_factory/core/pairs.py +144 -0
  58. data_factory/core/parallel.py +191 -0
  59. data_factory/core/provenance.py +136 -0
  60. data_factory/core/registry.py +339 -0
  61. data_factory/core/stats.py +403 -0
  62. data_factory/core/types.py +148 -0
  63. data_factory/core/writer.py +194 -0
  64. data_factory/desktop.py +207 -0
  65. data_factory/hardware.py +243 -0
  66. data_factory/pipeline.py +192 -0
  67. data_factory/py.typed +0 -0
  68. data_factory/service/__init__.py +26 -0
  69. data_factory/service/app.py +483 -0
  70. data_factory/service/binding.py +174 -0
  71. data_factory/service/deps.py +133 -0
  72. data_factory/service/jobs.py +177 -0
  73. data_factory/service/limits.py +71 -0
  74. data_factory/service/routers/__init__.py +11 -0
  75. data_factory/service/routers/annotate.py +582 -0
  76. data_factory/service/routers/local.py +464 -0
  77. data_factory/service/routers/media.py +154 -0
  78. data_factory/service/routers/predictions.py +116 -0
  79. data_factory/service/routers/promote.py +182 -0
  80. data_factory/service/routers/tools.py +308 -0
  81. data_factory/service/static/annotate/api.js +246 -0
  82. data_factory/service/static/annotate/app.js +1696 -0
  83. data_factory/service/static/annotate/assist.js +106 -0
  84. data_factory/service/static/annotate/filmstrip.js +286 -0
  85. data_factory/service/static/annotate/hit.js +146 -0
  86. data_factory/service/static/annotate/labels.js +197 -0
  87. data_factory/service/static/annotate/render.js +286 -0
  88. data_factory/service/static/annotate/scene.js +277 -0
  89. data_factory/service/static/annotate/shapelist.js +72 -0
  90. data_factory/service/static/annotate/tools/box.js +96 -0
  91. data_factory/service/static/annotate/tools/line.js +96 -0
  92. data_factory/service/static/annotate/tools/obb.js +88 -0
  93. data_factory/service/static/annotate/tools/point.js +49 -0
  94. data_factory/service/static/annotate/tools/polygon.js +110 -0
  95. data_factory/service/static/annotate/tools/select.js +227 -0
  96. data_factory/service/static/annotate/undo.js +176 -0
  97. data_factory/service/static/annotate/viewport.js +156 -0
  98. data_factory/service/static/annotate.html +390 -0
  99. data_factory/service/static/index.html +632 -0
  100. data_factory/service/workspace.py +244 -0
  101. easyannotate/__init__.py +64 -0
  102. easyannotate/__main__.py +5 -0
  103. easyannotate/project.py +181 -0
  104. easyannotate/py.typed +0 -0
  105. easyannotate-0.4.0.dist-info/METADATA +767 -0
  106. easyannotate-0.4.0.dist-info/RECORD +109 -0
  107. easyannotate-0.4.0.dist-info/WHEEL +4 -0
  108. easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
  109. easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,174 @@
1
+ """Turning a web request into operation parameters, without trusting paths.
2
+
3
+ Operations take real filesystem paths. A browser must never supply one, so this
4
+ module is the translation layer: the client picks *what* to run on by logical
5
+ name -- ``"source"``, or the name of an earlier output -- and every actual path
6
+ is derived from the dataset sandbox here.
7
+
8
+ The rule enforced below is blunt on purpose: **any path-kind parameter sent by
9
+ a client is rejected outright**, rather than sanitised. Sanitising invites an
10
+ arms race over encodings, symlinks and UNC paths; refusing does not.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from pathlib import Path
16
+ from typing import Any
17
+
18
+ from ..core.registry import Operation, ParamError
19
+ from .workspace import Dataset, Workspace, safe_segment
20
+
21
+ #: Parameter kinds that name a location on disk. None may come from a client.
22
+ PATH_KINDS = frozenset({"dir", "file", "path", "dirs"})
23
+
24
+ #: Parameters that name where an operation writes, rather than what it reads.
25
+ OUTPUT_NAMES = frozenset({"output", "quarantine"})
26
+
27
+
28
+ class BindingError(Exception):
29
+ """Raised when a request cannot be turned into a safe parameter set."""
30
+
31
+
32
+ def resolve_input(dataset: Dataset, reference: str | None, workspace: Workspace) -> Path:
33
+ """Map a logical input name onto a real directory inside the sandbox.
34
+
35
+ ``None`` or ``"source"`` means the uploaded dataset; anything else names an
36
+ output produced by an earlier step, which is what makes chaining possible
37
+ without ever exposing a path.
38
+ """
39
+ if reference in (None, "", "source"):
40
+ return workspace.assert_inside(dataset.source)
41
+ candidate = dataset.output(reference)
42
+ resolved = workspace.assert_inside(candidate)
43
+ if not resolved.is_dir():
44
+ raise BindingError(
45
+ f"no step has produced {reference!r} yet. Available: "
46
+ f"{', '.join(available_inputs(dataset)) or 'source'}"
47
+ )
48
+ return resolved
49
+
50
+
51
+ def available_inputs(dataset: Dataset) -> list[str]:
52
+ """Everything that can currently be used as an input, for the UI to offer."""
53
+ names = ["source"] if any(dataset.source.iterdir()) else []
54
+ if dataset.outputs.is_dir():
55
+ names.extend(sorted(p.name for p in dataset.outputs.iterdir() if p.is_dir()))
56
+ return names
57
+
58
+
59
+ def bind(
60
+ operation: Operation,
61
+ supplied: dict[str, Any],
62
+ *,
63
+ dataset: Dataset,
64
+ workspace: Workspace,
65
+ input_ref: str | None = None,
66
+ output_name: str = "",
67
+ ) -> tuple[dict[str, Any], Path | None]:
68
+ """Build a safe, fully-populated parameter set for one run.
69
+
70
+ Returns the parameters and the output directory, if the operation writes
71
+ one. Raises :class:`BindingError` for anything a client should not be able
72
+ to ask for, and lets :class:`ParamError` through for ordinary validation --
73
+ so the web API reports bad values exactly as the CLI does.
74
+ """
75
+ offered = dict(supplied or {})
76
+
77
+ # Refuse rather than sanitise. A rejected request is a bug report; a
78
+ # silently rewritten one is a vulnerability waiting for an encoding trick.
79
+ for spec in operation.params:
80
+ if spec.kind in PATH_KINDS and spec.name in offered:
81
+ raise BindingError(
82
+ f"{spec.name!r} is a path and cannot be set directly. Choose an "
83
+ f"input with 'input', and name the result with 'output_name'."
84
+ )
85
+
86
+ output_dir: Path | None = None
87
+ for spec in operation.params:
88
+ if spec.kind not in PATH_KINDS:
89
+ continue
90
+
91
+ if spec.name in OUTPUT_NAMES:
92
+ target = dataset.output(output_name or f"{operation.name.replace('.', '_')}")
93
+ output_dir = workspace.assert_inside(target)
94
+ if spec.kind == "file":
95
+ # report.html writes a file, not a directory.
96
+ output_dir.mkdir(parents=True, exist_ok=True)
97
+ offered[spec.name] = str(output_dir / _filename_for(operation))
98
+ else:
99
+ offered[spec.name] = str(output_dir)
100
+ continue
101
+
102
+ source = resolve_input(dataset, input_ref, workspace)
103
+ if spec.kind == "dirs":
104
+ offered[spec.name] = [str(source)]
105
+ elif spec.kind == "file":
106
+ offered[spec.name] = str(_find_file(source, operation))
107
+ else:
108
+ offered[spec.name] = str(source)
109
+
110
+ bound = operation.bind(offered)
111
+ return bound, output_dir
112
+
113
+
114
+ def _filename_for(operation: Operation) -> str:
115
+ return "report.html" if operation.name == "report.html" else "result.json"
116
+
117
+
118
+ def _find_file(folder: Path, operation: Operation) -> Path:
119
+ """Locate the single file a file-kind input wants.
120
+
121
+ Only ``inspect.coco`` needs this, and it wants an instances.json; guessing
122
+ is acceptable because the alternative is asking a browser for a path.
123
+ """
124
+ for pattern in ("**/instances*.json", "**/_annotations.coco.json", "**/*.json"):
125
+ for found in sorted(folder.glob(pattern)):
126
+ if found.is_file():
127
+ return found
128
+ raise BindingError(f"{operation.name} needs a json file and none was found in the dataset")
129
+
130
+
131
+ def describe_operation(operation: Operation) -> dict[str, Any]:
132
+ """The operation as a web form should render it.
133
+
134
+ Path parameters are stripped: the client cannot set them, so showing them
135
+ would only invite a request the service will refuse.
136
+ """
137
+ payload = operation.as_dict()
138
+ payload["params"] = [p for p in payload["params"] if p["kind"] not in PATH_KINDS]
139
+ payload["writes_output"] = any(
140
+ p.name in OUTPUT_NAMES for p in operation.params
141
+ )
142
+ return payload
143
+
144
+
145
+ def public_schema() -> dict[str, Any]:
146
+ """Every operation a browser can usefully render a form for.
147
+
148
+ The ``hidden_from_web`` ones are filtered here and only here -- they stay
149
+ in ``datafactory list``, in ``schema``, and in recipes. An operation whose
150
+ real input is an annotation project rather than a folder has nothing the
151
+ batch panel could ask for, so offering it would only produce a refusal.
152
+ """
153
+ from ..core import registry
154
+
155
+ return {
156
+ "version": 1,
157
+ "operations": [
158
+ describe_operation(op)
159
+ for op in registry.all_operations()
160
+ if not op.hidden_from_web
161
+ ],
162
+ }
163
+
164
+
165
+ __all__ = [
166
+ "BindingError",
167
+ "ParamError",
168
+ "available_inputs",
169
+ "bind",
170
+ "describe_operation",
171
+ "public_schema",
172
+ "resolve_input",
173
+ "safe_segment",
174
+ ]
@@ -0,0 +1,133 @@
1
+ """Shared request dependencies: the key gate, the store, and where files live.
2
+
3
+ Split out of ``app.py`` so the annotation routers can depend on the same gate
4
+ without importing the application module they are mounted into.
5
+
6
+ **Layout.** The dataset sandboxes live under the workspace root and are
7
+ transient -- swept on TTL, zipped for download, addressed by an opaque id.
8
+ Annotation projects are not transient, so nothing of theirs is stored in
9
+ there::
10
+
11
+ ~/.datafactory/
12
+ workspaces/ <- dataset sandboxes, swept, zipped, served
13
+ annotations.db <- the project database
14
+ projects/<id>/ <- project media
15
+
16
+ A database file inside a sandbox would eventually be swept out from under a
17
+ user, packed into somebody's download, or served by a static route. Putting it
18
+ one level up costs nothing and removes all three.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import os
24
+ import secrets
25
+ import threading
26
+ from pathlib import Path
27
+ from typing import Any
28
+
29
+ from fastapi import Header, HTTPException
30
+
31
+ from ..annotate.store import LOCAL_OWNER, Store
32
+ from .workspace import resolve_workspace_root
33
+
34
+ #: Read per call rather than captured at import, so a test (or a container
35
+ #: restart with new settings) does not need this module reloaded to see it.
36
+ def api_key() -> str:
37
+ return os.environ.get("DATAFACTORY_API_KEY", "")
38
+
39
+
40
+ async def require_key(x_api_key: str = Header(default="")) -> None:
41
+ """Optional shared-secret gate.
42
+
43
+ Off by default so the tool runs locally with no ceremony. Set
44
+ ``DATAFACTORY_API_KEY`` before exposing it to the internet -- without it,
45
+ anyone can spend your CPU and disk.
46
+ """
47
+ key = api_key()
48
+ if not key:
49
+ return
50
+ # isascii first: compare_digest raises on non-ASCII text -- a 500, not a 401.
51
+ if not x_api_key.isascii() or not secrets.compare_digest(x_api_key, key):
52
+ raise HTTPException(401, "invalid or missing X-API-Key")
53
+
54
+
55
+ _jobs: Any = None
56
+
57
+
58
+ def set_jobs_manager(manager: Any) -> None:
59
+ """The app registers its JobManager here so routers can submit work
60
+ without importing the app module (which imports the routers)."""
61
+ global _jobs
62
+ _jobs = manager
63
+
64
+
65
+ def jobs_manager() -> Any:
66
+ return _jobs
67
+
68
+
69
+ def data_root() -> Path:
70
+ """The directory holding the database and project media."""
71
+ return resolve_workspace_root().parent
72
+
73
+
74
+ def store_path() -> Path:
75
+ return data_root() / "annotations.db"
76
+
77
+
78
+ _stores: dict[Path, Store] = {}
79
+ _stores_lock = threading.Lock()
80
+
81
+
82
+ def get_store() -> Store:
83
+ """The annotation store for the configured workspace.
84
+
85
+ Cached *by path* rather than as a single global. Tests point
86
+ ``DATAFACTORY_WORKSPACE`` somewhere new between cases, and a plain global
87
+ would keep serving the first one they happened to create.
88
+ """
89
+ path = store_path()
90
+ with _stores_lock:
91
+ existing = _stores.get(path)
92
+ if existing is None:
93
+ existing = Store(path)
94
+ _stores[path] = existing
95
+ return existing
96
+
97
+
98
+ def reset_stores() -> None:
99
+ """Drop cached connections. For tests and for a clean shutdown."""
100
+ with _stores_lock:
101
+ for store in _stores.values():
102
+ store.close()
103
+ _stores.clear()
104
+
105
+
106
+ async def principal() -> str:
107
+ """Who is asking.
108
+
109
+ Locally this is always the same answer, and that is the point: every query
110
+ below is already scoped by owner, so Phase 5 replaces this one function
111
+ with a real session lookup instead of revisiting every route.
112
+ """
113
+ return LOCAL_OWNER
114
+
115
+
116
+ def media_root(project_id: str) -> Path:
117
+ """Where a managed project's images are kept."""
118
+ return data_root() / "projects" / project_id
119
+
120
+
121
+ def resolve_media(root: Path, rel_path: str) -> Path:
122
+ """Join a stored relative path onto a root, refusing anything that escapes.
123
+
124
+ The client never sends this path -- it names an image id and the path comes
125
+ out of the database. The check is here anyway: a row could have been
126
+ written by an older build, and a symlink planted under the root would
127
+ otherwise redirect the read.
128
+ """
129
+ base = root.resolve()
130
+ target = (base / rel_path).resolve()
131
+ if target != base and base not in target.parents:
132
+ raise HTTPException(400, "that image is outside the project")
133
+ return target
@@ -0,0 +1,177 @@
1
+ """Background jobs with progress and cancellation.
2
+
3
+ Dataset work outlives an HTTP request -- augmenting 3,000 images takes minutes,
4
+ and a browser will not hold a connection open for that. So a run returns a job
5
+ id immediately and the client polls.
6
+
7
+ This is a single-process, in-memory manager, which is the right size for one
8
+ container serving a modest site. It is deliberately shaped so that swapping in
9
+ Redis or Celery later means replacing this file and nothing else: the rest of
10
+ the service only ever calls ``submit``, ``get`` and ``cancel``.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import threading
16
+ import time
17
+ import traceback
18
+ from collections import OrderedDict
19
+ from collections.abc import Callable
20
+ from concurrent.futures import ThreadPoolExecutor
21
+ from dataclasses import dataclass, field
22
+ from pathlib import Path
23
+ from typing import Any
24
+
25
+ from ..core.types import Cancelled, Context, OpResult
26
+
27
+ #: Runs at once. Dataset work is CPU-bound and already multi-threaded inside
28
+ #: each operation, so letting several run concurrently mostly adds contention.
29
+ MAX_CONCURRENT = 2
30
+
31
+ #: Finished jobs kept for polling before being evicted.
32
+ MAX_REMEMBERED = 500
33
+
34
+
35
+ @dataclass
36
+ class Job:
37
+ id: str
38
+ operation: str
39
+ dataset_id: str
40
+ state: str = "queued" # queued | running | finished | failed | cancelled
41
+ done: int = 0
42
+ total: int = 0
43
+ message: str = ""
44
+ created_at: float = field(default_factory=time.time)
45
+ started_at: float = 0.0
46
+ finished_at: float = 0.0
47
+ result: dict[str, Any] | None = None
48
+ error: str = ""
49
+ output_name: str = ""
50
+
51
+ def as_dict(self) -> dict[str, Any]:
52
+ elapsed = (self.finished_at or time.time()) - (self.started_at or self.created_at)
53
+ return {
54
+ "id": self.id,
55
+ "operation": self.operation,
56
+ "dataset": self.dataset_id,
57
+ "state": self.state,
58
+ "progress": {
59
+ "done": self.done,
60
+ "total": self.total,
61
+ "message": self.message,
62
+ # A percentage the UI can render without dividing by zero.
63
+ "percent": round(100 * self.done / self.total, 1) if self.total else None,
64
+ },
65
+ "elapsed_seconds": round(elapsed, 1),
66
+ "output": self.output_name or None,
67
+ "result": self.result,
68
+ "error": self.error or None,
69
+ }
70
+
71
+
72
+ class JobManager:
73
+ """Runs operations off the request thread, with progress and cancellation."""
74
+
75
+ def __init__(self, max_concurrent: int = MAX_CONCURRENT) -> None:
76
+ self._pool = ThreadPoolExecutor(max_workers=max_concurrent, thread_name_prefix="dfx-job")
77
+ self._lock = threading.Lock()
78
+ self._jobs: OrderedDict[str, Job] = OrderedDict()
79
+ self._cancelled: set[str] = set()
80
+
81
+ # -- api ---------------------------------------------------------------
82
+ def submit(
83
+ self,
84
+ job_id: str,
85
+ operation: str,
86
+ dataset_id: str,
87
+ work: Callable[[Context], OpResult],
88
+ *,
89
+ output_name: str = "",
90
+ ) -> Job:
91
+ job = Job(id=job_id, operation=operation, dataset_id=dataset_id, output_name=output_name)
92
+ with self._lock:
93
+ self._jobs[job_id] = job
94
+ self._evict_locked()
95
+ self._pool.submit(self._run, job, work)
96
+ return job
97
+
98
+ def get(self, job_id: str) -> Job | None:
99
+ with self._lock:
100
+ return self._jobs.get(job_id)
101
+
102
+ def cancel(self, job_id: str) -> bool:
103
+ with self._lock:
104
+ job = self._jobs.get(job_id)
105
+ if job is None or job.state in ("finished", "failed", "cancelled"):
106
+ return False
107
+ self._cancelled.add(job_id)
108
+ if job.state == "queued":
109
+ # Never started, so nothing will poll the cancel flag for us.
110
+ job.state = "cancelled"
111
+ job.finished_at = time.time()
112
+ return True
113
+
114
+ def for_dataset(self, dataset_id: str) -> list[Job]:
115
+ with self._lock:
116
+ return [j for j in self._jobs.values() if j.dataset_id == dataset_id]
117
+
118
+ # -- internals ---------------------------------------------------------
119
+ def _run(self, job: Job, work: Callable[[Context], OpResult]) -> None:
120
+ with self._lock:
121
+ if job.id in self._cancelled:
122
+ job.state = "cancelled"
123
+ job.finished_at = time.time()
124
+ return
125
+ job.state = "running"
126
+ job.started_at = time.time()
127
+
128
+ def progress(done: int, total: int, message: str = "") -> None:
129
+ with self._lock:
130
+ job.done, job.total, job.message = done, total, message
131
+
132
+ def cancelled() -> bool:
133
+ with self._lock:
134
+ return job.id in self._cancelled
135
+
136
+ context = Context(progress=progress, cancel_check=cancelled)
137
+ try:
138
+ result = work(context)
139
+ with self._lock:
140
+ # Actions are the per-file audit trail: useful in a terminal,
141
+ # far too large to ship to a browser for a 10,000-file run.
142
+ job.result = result.as_dict(include_actions=False)
143
+ job.state = "finished" if result.ok else "failed"
144
+ if not result.ok:
145
+ job.error = "; ".join(result.errors) or "the operation reported failure"
146
+ job.finished_at = time.time()
147
+ except Cancelled:
148
+ with self._lock:
149
+ job.state = "cancelled"
150
+ job.finished_at = time.time()
151
+ except Exception as exc:
152
+ # The traceback goes to the log, not to the client: it can carry
153
+ # host paths and package layout that a public API should not leak.
154
+ traceback.print_exc()
155
+ with self._lock:
156
+ job.state = "failed"
157
+ job.error = f"{type(exc).__name__}: {exc}"
158
+ job.finished_at = time.time()
159
+ finally:
160
+ with self._lock:
161
+ self._cancelled.discard(job.id)
162
+
163
+ def _evict_locked(self) -> None:
164
+ while len(self._jobs) > MAX_REMEMBERED:
165
+ for key, job in self._jobs.items():
166
+ if job.state in ("finished", "failed", "cancelled"):
167
+ del self._jobs[key]
168
+ break
169
+ else:
170
+ return
171
+
172
+ def shutdown(self) -> None:
173
+ self._pool.shutdown(wait=False, cancel_futures=True)
174
+
175
+
176
+ def output_exists(path: Path | None) -> bool:
177
+ return bool(path and path.is_dir() and any(path.iterdir()))
@@ -0,0 +1,71 @@
1
+ """Request-size limits that act before a body is read.
2
+
3
+ A handler that counts bytes as it reads an ``UploadFile`` caps nothing:
4
+ Starlette has already parsed the multipart body -- spooling it to a temporary
5
+ file on disk -- before the handler runs. A 100 GB POST fills the disk first and
6
+ is refused afterwards. So the limit lives here, in front of the app, where the
7
+ only thing read is the header.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ from typing import Any
14
+
15
+ #: Room for multipart framing around a file that is itself within the limit.
16
+ FRAMING_SLACK = 1024 * 1024
17
+
18
+ _BODY_METHODS = {b"POST", b"PUT", b"PATCH"}
19
+
20
+
21
+ class BodyLimit:
22
+ """Refuse a request whose declared size exceeds ``limit`` -- before reading it.
23
+
24
+ A body with no ``Content-Length`` (chunked) is refused too, with 411:
25
+ without a declared length the only way to enforce a limit is to read the
26
+ body, which is the problem this exists to avoid. Browsers, httpx and curl
27
+ all declare the length of a file upload.
28
+ """
29
+
30
+ def __init__(self, app: Any, limit: int) -> None:
31
+ self.app = app
32
+ self.limit = int(limit) + FRAMING_SLACK
33
+
34
+ async def __call__(self, scope: dict, receive: Any, send: Any) -> None:
35
+ if scope["type"] != "http" or scope["method"].encode() not in _BODY_METHODS:
36
+ await self.app(scope, receive, send)
37
+ return
38
+
39
+ headers = dict(scope.get("headers") or [])
40
+ declared = headers.get(b"content-length")
41
+ if declared is None:
42
+ if headers.get(b"transfer-encoding", b"").lower() == b"chunked":
43
+ await _refuse(send, 411, "send a Content-Length with uploads")
44
+ return
45
+ elif not declared.isdigit():
46
+ await _refuse(send, 400, "Content-Length is not a number")
47
+ return
48
+ elif int(declared) > self.limit:
49
+ mb = (self.limit - FRAMING_SLACK) / 1e6
50
+ await _refuse(send, 413, f"upload exceeds the {mb:.0f} MB limit")
51
+ return
52
+
53
+ await self.app(scope, receive, send)
54
+
55
+
56
+ async def _refuse(send: Any, status: int, detail: str) -> None:
57
+ body = json.dumps({"detail": detail}).encode()
58
+ await send(
59
+ {
60
+ "type": "http.response.start",
61
+ "status": status,
62
+ "headers": [
63
+ (b"content-type", b"application/json"),
64
+ (b"content-length", str(len(body)).encode()),
65
+ # The client may still be sending; tell it not to reuse a
66
+ # connection with an unread body on it.
67
+ (b"connection", b"close"),
68
+ ],
69
+ }
70
+ )
71
+ await send({"type": "http.response.body", "body": body})
@@ -0,0 +1,11 @@
1
+ """HTTP routers.
2
+
3
+ The batch API in ``app.py`` is request/response over folders. Annotation is
4
+ stateful CRUD with a latency budget, so it lives here instead of being forced
5
+ through the operation registry -- see ``data_factory/annotate/__init__.py`` for
6
+ why that boundary exists.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ __all__ = ["annotate", "media", "predictions", "promote", "tools"]