easyannotate 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_factory/__init__.py +82 -0
- data_factory/__main__.py +6 -0
- data_factory/annotate/__init__.py +15 -0
- data_factory/annotate/attach.py +398 -0
- data_factory/annotate/demo.py +161 -0
- data_factory/annotate/images.py +443 -0
- data_factory/annotate/labels.py +399 -0
- data_factory/annotate/loop.py +359 -0
- data_factory/annotate/projects.py +207 -0
- data_factory/annotate/review.py +471 -0
- data_factory/annotate/schema.sql +171 -0
- data_factory/annotate/shapes.py +488 -0
- data_factory/annotate/store.py +292 -0
- data_factory/annotate/tools/__init__.py +39 -0
- data_factory/annotate/tools/fake.py +170 -0
- data_factory/annotate/tools/install.py +254 -0
- data_factory/annotate/tools/local.py +389 -0
- data_factory/annotate/tools/postprocess.py +175 -0
- data_factory/annotate/tools/protocol.py +238 -0
- data_factory/annotate/tools/registry.py +196 -0
- data_factory/annotate/tools/results.py +83 -0
- data_factory/annotate/tools/serve.py +106 -0
- data_factory/cli/__init__.py +21 -0
- data_factory/cli/main.py +837 -0
- data_factory/cli/menu.py +173 -0
- data_factory/cli/render.py +158 -0
- data_factory/core/__init__.py +11 -0
- data_factory/core/annotations/__init__.py +5 -0
- data_factory/core/annotations/coco.py +394 -0
- data_factory/core/annotations/csvfmt.py +174 -0
- data_factory/core/annotations/dota.py +98 -0
- data_factory/core/annotations/formats.py +626 -0
- data_factory/core/annotations/labelme.py +153 -0
- data_factory/core/annotations/model.py +334 -0
- data_factory/core/annotations/voc.py +231 -0
- data_factory/core/annotations/yolo.py +283 -0
- data_factory/core/bktree.py +89 -0
- data_factory/core/fs.py +245 -0
- data_factory/core/imaging.py +626 -0
- data_factory/core/ops/__init__.py +44 -0
- data_factory/core/ops/analyze.py +383 -0
- data_factory/core/ops/annotate_export.py +326 -0
- data_factory/core/ops/audit.py +274 -0
- data_factory/core/ops/augment.py +454 -0
- data_factory/core/ops/collect.py +241 -0
- data_factory/core/ops/combine.py +582 -0
- data_factory/core/ops/convert.py +212 -0
- data_factory/core/ops/dataset.py +246 -0
- data_factory/core/ops/dedup.py +235 -0
- data_factory/core/ops/export.py +168 -0
- data_factory/core/ops/image.py +407 -0
- data_factory/core/ops/labels.py +215 -0
- data_factory/core/ops/pairing.py +133 -0
- data_factory/core/ops/rename.py +154 -0
- data_factory/core/ops/report.py +323 -0
- data_factory/core/ops/visualize.py +164 -0
- data_factory/core/pairs.py +144 -0
- data_factory/core/parallel.py +191 -0
- data_factory/core/provenance.py +136 -0
- data_factory/core/registry.py +339 -0
- data_factory/core/stats.py +403 -0
- data_factory/core/types.py +148 -0
- data_factory/core/writer.py +194 -0
- data_factory/desktop.py +207 -0
- data_factory/hardware.py +243 -0
- data_factory/pipeline.py +192 -0
- data_factory/py.typed +0 -0
- data_factory/service/__init__.py +26 -0
- data_factory/service/app.py +483 -0
- data_factory/service/binding.py +174 -0
- data_factory/service/deps.py +133 -0
- data_factory/service/jobs.py +177 -0
- data_factory/service/limits.py +71 -0
- data_factory/service/routers/__init__.py +11 -0
- data_factory/service/routers/annotate.py +582 -0
- data_factory/service/routers/local.py +464 -0
- data_factory/service/routers/media.py +154 -0
- data_factory/service/routers/predictions.py +116 -0
- data_factory/service/routers/promote.py +182 -0
- data_factory/service/routers/tools.py +308 -0
- data_factory/service/static/annotate/api.js +246 -0
- data_factory/service/static/annotate/app.js +1696 -0
- data_factory/service/static/annotate/assist.js +106 -0
- data_factory/service/static/annotate/filmstrip.js +286 -0
- data_factory/service/static/annotate/hit.js +146 -0
- data_factory/service/static/annotate/labels.js +197 -0
- data_factory/service/static/annotate/render.js +286 -0
- data_factory/service/static/annotate/scene.js +277 -0
- data_factory/service/static/annotate/shapelist.js +72 -0
- data_factory/service/static/annotate/tools/box.js +96 -0
- data_factory/service/static/annotate/tools/line.js +96 -0
- data_factory/service/static/annotate/tools/obb.js +88 -0
- data_factory/service/static/annotate/tools/point.js +49 -0
- data_factory/service/static/annotate/tools/polygon.js +110 -0
- data_factory/service/static/annotate/tools/select.js +227 -0
- data_factory/service/static/annotate/undo.js +176 -0
- data_factory/service/static/annotate/viewport.js +156 -0
- data_factory/service/static/annotate.html +390 -0
- data_factory/service/static/index.html +632 -0
- data_factory/service/workspace.py +244 -0
- easyannotate/__init__.py +64 -0
- easyannotate/__main__.py +5 -0
- easyannotate/project.py +181 -0
- easyannotate/py.typed +0 -0
- easyannotate-0.4.0.dist-info/METADATA +767 -0
- easyannotate-0.4.0.dist-info/RECORD +109 -0
- easyannotate-0.4.0.dist-info/WHEEL +4 -0
- easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
- easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Parallel map for the decode-bound stages.
|
|
2
|
+
|
|
3
|
+
Perceptual hashing 2,783 images took 67 seconds, and essentially all of it was
|
|
4
|
+
JPEG decode -- one core busy, the rest idle. Decoding is embarrassingly
|
|
5
|
+
parallel, so this is the piece that was missing rather than a smarter
|
|
6
|
+
algorithm; the BK-tree already fixed the comparison cost.
|
|
7
|
+
|
|
8
|
+
Threads, not processes. OpenCV and Pillow release the GIL around decode, so
|
|
9
|
+
threads get real parallelism here, and they avoid pickling a few thousand image
|
|
10
|
+
paths and hash objects across a process boundary. A process pool would also
|
|
11
|
+
break the Windows entry-point rules for a library that gets imported by a web
|
|
12
|
+
server.
|
|
13
|
+
|
|
14
|
+
Two properties matter more than the speed:
|
|
15
|
+
|
|
16
|
+
*Order is preserved.* Dedup keeps the first image it sees, so results that
|
|
17
|
+
depended on which thread finished first would not be reproducible. Results come
|
|
18
|
+
back in input order regardless of completion order.
|
|
19
|
+
|
|
20
|
+
*Cancellation and progress still work.* Both are polled from the calling
|
|
21
|
+
thread as results arrive, so a web request can still abort a long run.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import os
|
|
27
|
+
from collections import deque
|
|
28
|
+
from collections.abc import Callable, Iterator, Sequence
|
|
29
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
30
|
+
from typing import Any, TypeVar
|
|
31
|
+
|
|
32
|
+
from .types import Context
|
|
33
|
+
|
|
34
|
+
T = TypeVar("T")
|
|
35
|
+
R = TypeVar("R")
|
|
36
|
+
|
|
37
|
+
#: Past this, threads contend on memory bandwidth rather than adding speed, and
|
|
38
|
+
#: a 32-core machine should not open 32 large JPEGs at once.
|
|
39
|
+
MAX_WORKERS = 8
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def resolve_workers(requested: int) -> int:
|
|
43
|
+
"""Turn a ``--workers`` value into an actual thread count.
|
|
44
|
+
|
|
45
|
+
0 means "decide for me": one per core, capped, and always leaving the
|
|
46
|
+
machine usable.
|
|
47
|
+
"""
|
|
48
|
+
if requested and requested > 0:
|
|
49
|
+
return max(1, min(int(requested), 32))
|
|
50
|
+
cores = os.cpu_count() or 2
|
|
51
|
+
return max(1, min(cores - 1, MAX_WORKERS))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def map_ordered(
|
|
55
|
+
items: Sequence[T],
|
|
56
|
+
work: Callable[[T], R],
|
|
57
|
+
*,
|
|
58
|
+
ctx: Context,
|
|
59
|
+
workers: int = 0,
|
|
60
|
+
label: Callable[[T], str] | None = None,
|
|
61
|
+
on_error: Callable[[T, Exception], None] | None = None,
|
|
62
|
+
) -> Iterator[tuple[T, R | None]]:
|
|
63
|
+
"""Apply ``work`` to every item, yielding ``(item, result)`` in input order.
|
|
64
|
+
|
|
65
|
+
Items whose work raised yield ``None`` after ``on_error`` is called, so a
|
|
66
|
+
single corrupt file cannot abort a scan of twenty thousand.
|
|
67
|
+
"""
|
|
68
|
+
total = len(items)
|
|
69
|
+
if total == 0:
|
|
70
|
+
return
|
|
71
|
+
|
|
72
|
+
count = resolve_workers(workers)
|
|
73
|
+
if count == 1 or total == 1:
|
|
74
|
+
# Serial path: identical semantics, no thread pool overhead, and a much
|
|
75
|
+
# easier thing to step through in a debugger.
|
|
76
|
+
for position, item in enumerate(items, start=1):
|
|
77
|
+
ctx.check_cancelled()
|
|
78
|
+
ctx.emit(position, total, label(item) if label else "")
|
|
79
|
+
try:
|
|
80
|
+
yield item, work(item)
|
|
81
|
+
except Exception as exc:
|
|
82
|
+
if on_error is not None:
|
|
83
|
+
on_error(item, exc)
|
|
84
|
+
yield item, None
|
|
85
|
+
return
|
|
86
|
+
|
|
87
|
+
with ThreadPoolExecutor(max_workers=count) as pool:
|
|
88
|
+
futures = [pool.submit(work, item) for item in items]
|
|
89
|
+
for position, (item, future) in enumerate(zip(items, futures), start=1):
|
|
90
|
+
try:
|
|
91
|
+
ctx.check_cancelled()
|
|
92
|
+
except BaseException:
|
|
93
|
+
# Stop the queue draining into work nobody will read. Already
|
|
94
|
+
# running tasks finish; the rest never start.
|
|
95
|
+
for pending in futures[position - 1 :]:
|
|
96
|
+
pending.cancel()
|
|
97
|
+
raise
|
|
98
|
+
ctx.emit(position, total, label(item) if label else "")
|
|
99
|
+
try:
|
|
100
|
+
yield item, future.result()
|
|
101
|
+
except Exception as exc:
|
|
102
|
+
if on_error is not None:
|
|
103
|
+
on_error(item, exc)
|
|
104
|
+
yield item, None
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def map_bounded(
|
|
108
|
+
items: Sequence[T],
|
|
109
|
+
work: Callable[[T], R],
|
|
110
|
+
*,
|
|
111
|
+
ctx: Context,
|
|
112
|
+
workers: int = 0,
|
|
113
|
+
label: Callable[[T], str] | None = None,
|
|
114
|
+
on_error: Callable[[T, Exception], None] | None = None,
|
|
115
|
+
in_flight: int = 0,
|
|
116
|
+
) -> Iterator[tuple[T, R | None]]:
|
|
117
|
+
""":func:`map_ordered`, but with a bounded number of tasks in flight.
|
|
118
|
+
|
|
119
|
+
The difference matters when the work returns something *large*.
|
|
120
|
+
``map_ordered`` submits every task up front, which is right for hashing --
|
|
121
|
+
a hash is 8 bytes -- and catastrophic for anything returning decoded or
|
|
122
|
+
encoded image data: 3,686 frames would all be resident at once.
|
|
123
|
+
|
|
124
|
+
Here at most ``in_flight`` results exist simultaneously, so memory stays
|
|
125
|
+
flat regardless of dataset size. Order is preserved exactly as before.
|
|
126
|
+
"""
|
|
127
|
+
total = len(items)
|
|
128
|
+
if total == 0:
|
|
129
|
+
return
|
|
130
|
+
|
|
131
|
+
count = resolve_workers(workers)
|
|
132
|
+
if count == 1 or total == 1:
|
|
133
|
+
yield from map_ordered(
|
|
134
|
+
items, work, ctx=ctx, workers=1, label=label, on_error=on_error
|
|
135
|
+
)
|
|
136
|
+
return
|
|
137
|
+
|
|
138
|
+
limit = in_flight if in_flight > 0 else max(2, count * 2)
|
|
139
|
+
pending: deque[tuple[T, Any]] = deque()
|
|
140
|
+
position = 0
|
|
141
|
+
|
|
142
|
+
def take() -> tuple[T, R | None]:
|
|
143
|
+
nonlocal position
|
|
144
|
+
item, future = pending.popleft()
|
|
145
|
+
position += 1
|
|
146
|
+
ctx.emit(position, total, label(item) if label else "")
|
|
147
|
+
try:
|
|
148
|
+
return item, future.result()
|
|
149
|
+
except Exception as exc:
|
|
150
|
+
if on_error is not None:
|
|
151
|
+
on_error(item, exc)
|
|
152
|
+
return item, None
|
|
153
|
+
|
|
154
|
+
with ThreadPoolExecutor(max_workers=count) as pool:
|
|
155
|
+
try:
|
|
156
|
+
for item in items:
|
|
157
|
+
ctx.check_cancelled()
|
|
158
|
+
pending.append((item, pool.submit(work, item)))
|
|
159
|
+
if len(pending) >= limit:
|
|
160
|
+
yield take()
|
|
161
|
+
while pending:
|
|
162
|
+
ctx.check_cancelled()
|
|
163
|
+
yield take()
|
|
164
|
+
except BaseException:
|
|
165
|
+
for _, future in pending:
|
|
166
|
+
future.cancel()
|
|
167
|
+
raise
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def collect(
|
|
171
|
+
items: Sequence[T],
|
|
172
|
+
work: Callable[[T], R],
|
|
173
|
+
*,
|
|
174
|
+
ctx: Context,
|
|
175
|
+
workers: int = 0,
|
|
176
|
+
label: Callable[[T], str] | None = None,
|
|
177
|
+
on_error: Callable[[T, Exception], None] | None = None,
|
|
178
|
+
) -> list[tuple[T, R]]:
|
|
179
|
+
""":func:`map_ordered` with the failures dropped, as a list."""
|
|
180
|
+
return [
|
|
181
|
+
(item, value)
|
|
182
|
+
for item, value in map_ordered(
|
|
183
|
+
items, work, ctx=ctx, workers=workers, label=label, on_error=on_error
|
|
184
|
+
)
|
|
185
|
+
if value is not None
|
|
186
|
+
]
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
WORKERS_HELP = (
|
|
190
|
+
"Parallel worker threads for image decoding (0 = choose automatically)."
|
|
191
|
+
)
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""Dataset lineage: what was done to this folder, and with what settings.
|
|
2
|
+
|
|
3
|
+
Six weeks after a dataset is built, the question that matters is "what is this,
|
|
4
|
+
exactly" -- was it augmented, at what seed, was it resized before or after the
|
|
5
|
+
split, which source folders went into the merge. Without a record that question
|
|
6
|
+
is unanswerable, and the usual answer is to rebuild the dataset from scratch
|
|
7
|
+
and hope.
|
|
8
|
+
|
|
9
|
+
So every operation that writes a dataset appends one line to
|
|
10
|
+
``<output>/.datafactory/history.jsonl``. It is recorded in
|
|
11
|
+
:meth:`Operation.run`, not by the operations themselves, which means no
|
|
12
|
+
operation can forget to do it and no new operation has to remember.
|
|
13
|
+
|
|
14
|
+
JSON Lines rather than a single JSON document, because appending must not
|
|
15
|
+
require reading and rewriting the file -- a build that dies halfway still
|
|
16
|
+
leaves a readable, truthful history of what it managed to do.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
from datetime import UTC, datetime
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Any
|
|
26
|
+
|
|
27
|
+
from .types import OpResult, _jsonable
|
|
28
|
+
|
|
29
|
+
HISTORY_DIR = ".datafactory"
|
|
30
|
+
HISTORY_FILE = "history.jsonl"
|
|
31
|
+
|
|
32
|
+
#: Parameters naming where output goes. Whichever is present decides which
|
|
33
|
+
#: folder gets the history entry.
|
|
34
|
+
OUTPUT_KEYS = ("output", "quarantine")
|
|
35
|
+
|
|
36
|
+
#: Never copy these into a record -- they are noise, or they are large.
|
|
37
|
+
SKIP_KEYS = frozenset({"workers", "quality", "recursive"})
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def history_path(folder: Path) -> Path:
|
|
41
|
+
return Path(folder) / HISTORY_DIR / HISTORY_FILE
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def record(
|
|
45
|
+
op_name: str,
|
|
46
|
+
params: dict[str, Any],
|
|
47
|
+
result: OpResult,
|
|
48
|
+
*,
|
|
49
|
+
version: str = "",
|
|
50
|
+
) -> Path | None:
|
|
51
|
+
"""Append one entry to the output folder's history. Returns the file written.
|
|
52
|
+
|
|
53
|
+
Silent no-op when the run was a dry run, failed, or produced no output
|
|
54
|
+
folder -- there is nothing to describe the provenance *of*.
|
|
55
|
+
"""
|
|
56
|
+
if result.dry_run or not result.ok:
|
|
57
|
+
return None
|
|
58
|
+
folder = _output_folder(params)
|
|
59
|
+
if folder is None or not folder.is_dir():
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
entry = {
|
|
63
|
+
"at": datetime.now(UTC).isoformat(timespec="seconds"),
|
|
64
|
+
"op": op_name,
|
|
65
|
+
"version": version,
|
|
66
|
+
"params": _describe(params),
|
|
67
|
+
"counts": dict(result.counts),
|
|
68
|
+
"warnings": len(result.warnings),
|
|
69
|
+
"produced": _jsonable(result.data.get("format")) or None,
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
target = history_path(folder)
|
|
73
|
+
try:
|
|
74
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
75
|
+
with target.open("a", encoding="utf-8") as handle:
|
|
76
|
+
handle.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
|
77
|
+
except OSError:
|
|
78
|
+
# Provenance is valuable but never worth failing a completed build for.
|
|
79
|
+
return None
|
|
80
|
+
return target
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def read(folder: Path) -> list[dict[str, Any]]:
|
|
84
|
+
"""Read a folder's history, skipping any line that got truncated."""
|
|
85
|
+
path = history_path(folder)
|
|
86
|
+
if not path.is_file():
|
|
87
|
+
return []
|
|
88
|
+
entries: list[dict[str, Any]] = []
|
|
89
|
+
for line in path.read_text(encoding="utf-8").splitlines():
|
|
90
|
+
line = line.strip()
|
|
91
|
+
if not line:
|
|
92
|
+
continue
|
|
93
|
+
try:
|
|
94
|
+
entries.append(json.loads(line))
|
|
95
|
+
except json.JSONDecodeError:
|
|
96
|
+
continue
|
|
97
|
+
return entries
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _output_folder(params: dict[str, Any]) -> Path | None:
|
|
101
|
+
for key in OUTPUT_KEYS:
|
|
102
|
+
value = params.get(key)
|
|
103
|
+
if isinstance(value, Path):
|
|
104
|
+
return value
|
|
105
|
+
return None
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _describe(params: dict[str, Any]) -> dict[str, Any]:
|
|
109
|
+
"""The settings worth keeping, as JSON.
|
|
110
|
+
|
|
111
|
+
Defaults are kept as well as overrides: "what seed was this augmented at"
|
|
112
|
+
has to be answerable without also knowing what the default was in the
|
|
113
|
+
version that built it.
|
|
114
|
+
"""
|
|
115
|
+
described: dict[str, Any] = {}
|
|
116
|
+
for key, value in params.items():
|
|
117
|
+
if key in SKIP_KEYS or value is None:
|
|
118
|
+
continue
|
|
119
|
+
if isinstance(value, Path):
|
|
120
|
+
described[key] = _short(value)
|
|
121
|
+
elif isinstance(value, (list, tuple)):
|
|
122
|
+
described[key] = [_short(v) if isinstance(v, Path) else v for v in value]
|
|
123
|
+
else:
|
|
124
|
+
described[key] = _jsonable(value)
|
|
125
|
+
return described
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _short(path: Path) -> str:
|
|
129
|
+
"""Absolute paths, but relative to the working directory when that is
|
|
130
|
+
shorter -- a history full of D:/long/absolute/prefixes is unreadable."""
|
|
131
|
+
text = str(path)
|
|
132
|
+
try:
|
|
133
|
+
relative = os.path.relpath(path, Path.cwd())
|
|
134
|
+
except (ValueError, OSError):
|
|
135
|
+
return text
|
|
136
|
+
return relative if len(relative) < len(text) and not relative.startswith("..") else text
|
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
"""The operation registry: one declaration, four front-ends.
|
|
2
|
+
|
|
3
|
+
Every operation states its own parameters once. From that single declaration we
|
|
4
|
+
generate the argparse subcommand, the interactive menu prompts, the YAML step
|
|
5
|
+
validator, and the JSON schema a web app turns into a form. Adding an operation
|
|
6
|
+
therefore means writing one file -- it appears everywhere else for free, and no
|
|
7
|
+
front-end has to be edited.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from collections.abc import Callable, Iterable
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from .types import Context, OpResult
|
|
18
|
+
|
|
19
|
+
#: Parameter kinds. ``dir``/``file``/``path`` are all strings at runtime but
|
|
20
|
+
#: carry different intent -- a web form renders a folder picker for ``dir`` and
|
|
21
|
+
#: the CLI can validate existence before an operation starts.
|
|
22
|
+
PARAM_KINDS = (
|
|
23
|
+
"dir",
|
|
24
|
+
"file",
|
|
25
|
+
"path",
|
|
26
|
+
"str",
|
|
27
|
+
"int",
|
|
28
|
+
"float",
|
|
29
|
+
"bool",
|
|
30
|
+
"choice",
|
|
31
|
+
"dirs",
|
|
32
|
+
"strs",
|
|
33
|
+
"mapping",
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class ParamError(ValueError):
|
|
38
|
+
"""Raised when supplied parameters are missing, unknown, or ill-typed."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(slots=True)
|
|
42
|
+
class Param:
|
|
43
|
+
name: str
|
|
44
|
+
kind: str
|
|
45
|
+
help: str
|
|
46
|
+
default: Any = None
|
|
47
|
+
required: bool = False
|
|
48
|
+
choices: tuple[str, ...] | None = None
|
|
49
|
+
minimum: float | None = None
|
|
50
|
+
maximum: float | None = None
|
|
51
|
+
must_exist: bool = False
|
|
52
|
+
|
|
53
|
+
def __post_init__(self) -> None:
|
|
54
|
+
if self.kind not in PARAM_KINDS:
|
|
55
|
+
raise ValueError(f"{self.name}: unknown param kind {self.kind!r}")
|
|
56
|
+
if self.kind == "choice" and not self.choices:
|
|
57
|
+
raise ValueError(f"{self.name}: choice params need choices")
|
|
58
|
+
|
|
59
|
+
# -- coercion ----------------------------------------------------------
|
|
60
|
+
def coerce(self, raw: Any, *, check_exists: bool = True) -> Any:
|
|
61
|
+
"""Turn a string (CLI/YAML/HTTP form) into the real Python value.
|
|
62
|
+
|
|
63
|
+
``check_exists`` is switched off when validating a recipe up front: a
|
|
64
|
+
later step's input folder is legitimately created by an earlier step,
|
|
65
|
+
so it will not exist yet at validation time.
|
|
66
|
+
"""
|
|
67
|
+
if raw is None:
|
|
68
|
+
return None
|
|
69
|
+
verify = self.must_exist and check_exists
|
|
70
|
+
if self.kind in ("dir", "file", "path"):
|
|
71
|
+
path = Path(str(raw)).expanduser()
|
|
72
|
+
if verify and not path.exists():
|
|
73
|
+
raise ParamError(f"{self.name}: path does not exist: {path}")
|
|
74
|
+
if verify and self.kind == "dir" and not path.is_dir():
|
|
75
|
+
raise ParamError(f"{self.name}: not a directory: {path}")
|
|
76
|
+
return path
|
|
77
|
+
if self.kind == "dirs":
|
|
78
|
+
items = raw if isinstance(raw, (list, tuple)) else _split_list(str(raw))
|
|
79
|
+
paths = [Path(str(i)).expanduser() for i in items]
|
|
80
|
+
if verify:
|
|
81
|
+
missing = [str(p) for p in paths if not p.is_dir()]
|
|
82
|
+
if missing:
|
|
83
|
+
raise ParamError(f"{self.name}: missing folders: {', '.join(missing)}")
|
|
84
|
+
return paths
|
|
85
|
+
if self.kind == "strs":
|
|
86
|
+
if isinstance(raw, (list, tuple)):
|
|
87
|
+
return [str(i) for i in raw]
|
|
88
|
+
return _split_list(str(raw))
|
|
89
|
+
if self.kind == "mapping":
|
|
90
|
+
return _coerce_mapping(self.name, raw)
|
|
91
|
+
if self.kind == "int":
|
|
92
|
+
value = int(raw)
|
|
93
|
+
self._check_range(value)
|
|
94
|
+
return value
|
|
95
|
+
if self.kind == "float":
|
|
96
|
+
number = float(raw)
|
|
97
|
+
self._check_range(number)
|
|
98
|
+
return number
|
|
99
|
+
if self.kind == "bool":
|
|
100
|
+
if isinstance(raw, bool):
|
|
101
|
+
return raw
|
|
102
|
+
text = str(raw).strip().lower()
|
|
103
|
+
if text in ("1", "true", "yes", "y", "on"):
|
|
104
|
+
return True
|
|
105
|
+
if text in ("0", "false", "no", "n", "off"):
|
|
106
|
+
return False
|
|
107
|
+
raise ParamError(f"{self.name}: expected a boolean, got {raw!r}")
|
|
108
|
+
if self.kind == "choice":
|
|
109
|
+
text = str(raw)
|
|
110
|
+
assert self.choices is not None
|
|
111
|
+
if text not in self.choices:
|
|
112
|
+
raise ParamError(
|
|
113
|
+
f"{self.name}: {text!r} is not one of {', '.join(self.choices)}"
|
|
114
|
+
)
|
|
115
|
+
return text
|
|
116
|
+
return str(raw)
|
|
117
|
+
|
|
118
|
+
def _check_range(self, value: float) -> None:
|
|
119
|
+
if self.minimum is not None and value < self.minimum:
|
|
120
|
+
raise ParamError(f"{self.name}: must be >= {self.minimum}")
|
|
121
|
+
if self.maximum is not None and value > self.maximum:
|
|
122
|
+
raise ParamError(f"{self.name}: must be <= {self.maximum}")
|
|
123
|
+
|
|
124
|
+
# -- introspection -----------------------------------------------------
|
|
125
|
+
def as_dict(self) -> dict[str, Any]:
|
|
126
|
+
out: dict[str, Any] = {
|
|
127
|
+
"name": self.name,
|
|
128
|
+
"kind": self.kind,
|
|
129
|
+
"help": self.help,
|
|
130
|
+
"required": self.required,
|
|
131
|
+
"default": str(self.default) if isinstance(self.default, Path) else self.default,
|
|
132
|
+
}
|
|
133
|
+
if self.choices:
|
|
134
|
+
out["choices"] = list(self.choices)
|
|
135
|
+
if self.minimum is not None:
|
|
136
|
+
out["minimum"] = self.minimum
|
|
137
|
+
if self.maximum is not None:
|
|
138
|
+
out["maximum"] = self.maximum
|
|
139
|
+
return out
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _split_list(text: str) -> list[str]:
|
|
143
|
+
"""Split a comma- or semicolon-separated list, tolerating stray spaces."""
|
|
144
|
+
parts = [p.strip() for chunk in text.split(";") for p in chunk.split(",")]
|
|
145
|
+
return [p for p in parts if p]
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _coerce_mapping(name: str, raw: Any) -> dict[str, str]:
|
|
149
|
+
"""Accept a dict (YAML/JSON) or an ``old=new,old2=new2`` string (CLI)."""
|
|
150
|
+
if isinstance(raw, dict):
|
|
151
|
+
return {str(k): str(v) for k, v in raw.items()}
|
|
152
|
+
mapping: dict[str, str] = {}
|
|
153
|
+
for item in _split_list(str(raw)):
|
|
154
|
+
if "=" not in item:
|
|
155
|
+
raise ParamError(f"{name}: expected old=new pairs, got {item!r}")
|
|
156
|
+
old, new = item.split("=", 1)
|
|
157
|
+
mapping[old.strip()] = new.strip()
|
|
158
|
+
return mapping
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
OpFunc = Callable[[dict[str, Any], Context], OpResult]
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@dataclass(slots=True)
|
|
165
|
+
class Operation:
|
|
166
|
+
"""A single unit of work, plus everything a front-end needs to drive it."""
|
|
167
|
+
|
|
168
|
+
name: str
|
|
169
|
+
title: str
|
|
170
|
+
category: str
|
|
171
|
+
summary: str
|
|
172
|
+
params: tuple[Param, ...]
|
|
173
|
+
func: OpFunc
|
|
174
|
+
destructive: bool = False
|
|
175
|
+
#: Longer prose shown by ``datafactory explain <op>`` and web tooltips.
|
|
176
|
+
notes: str = ""
|
|
177
|
+
#: Given bound parameters, return True when this particular run changes
|
|
178
|
+
#: nothing -- ``--action report`` on an otherwise destructive operation, say.
|
|
179
|
+
safe_when: Callable[[dict[str, Any]], bool] | None = None
|
|
180
|
+
#: Keep this operation out of the web form list while leaving it in
|
|
181
|
+
#: ``datafactory list`` and in recipes. For operations whose real inputs
|
|
182
|
+
#: are not a folder the browser can name -- an annotation project, say --
|
|
183
|
+
#: so rendering a form for them would only invite a refused request.
|
|
184
|
+
hidden_from_web: bool = False
|
|
185
|
+
|
|
186
|
+
def destructive_for(self, params: dict[str, Any]) -> bool:
|
|
187
|
+
"""Whether *this* invocation, with these parameters, writes anything."""
|
|
188
|
+
if not self.destructive:
|
|
189
|
+
return False
|
|
190
|
+
if self.safe_when is not None and self.safe_when(params):
|
|
191
|
+
return False
|
|
192
|
+
return True
|
|
193
|
+
|
|
194
|
+
def param(self, name: str) -> Param:
|
|
195
|
+
for candidate in self.params:
|
|
196
|
+
if candidate.name == name:
|
|
197
|
+
return candidate
|
|
198
|
+
raise KeyError(name)
|
|
199
|
+
|
|
200
|
+
def bind(self, supplied: dict[str, Any], *, check_paths: bool = True) -> dict[str, Any]:
|
|
201
|
+
"""Validate supplied values, apply defaults, and coerce types.
|
|
202
|
+
|
|
203
|
+
Every front-end funnels through here, so a YAML recipe and an HTTP POST
|
|
204
|
+
get byte-identical validation to a typed command line.
|
|
205
|
+
|
|
206
|
+
``check_paths=False`` validates everything except whether paths exist --
|
|
207
|
+
what a recipe needs at load time, since step 7's input is step 6's output.
|
|
208
|
+
"""
|
|
209
|
+
known = {p.name for p in self.params}
|
|
210
|
+
unknown = set(supplied) - known
|
|
211
|
+
if unknown:
|
|
212
|
+
raise ParamError(
|
|
213
|
+
f"{self.name}: unknown parameter(s): {', '.join(sorted(unknown))}. "
|
|
214
|
+
f"Valid: {', '.join(sorted(known))}"
|
|
215
|
+
)
|
|
216
|
+
bound: dict[str, Any] = {}
|
|
217
|
+
for spec in self.params:
|
|
218
|
+
raw = supplied.get(spec.name)
|
|
219
|
+
if raw is None or raw == "":
|
|
220
|
+
if spec.required:
|
|
221
|
+
raise ParamError(
|
|
222
|
+
f"{self.name}: missing required parameter {spec.name!r} ({spec.help})"
|
|
223
|
+
)
|
|
224
|
+
bound[spec.name] = (
|
|
225
|
+
spec.coerce(spec.default, check_exists=check_paths)
|
|
226
|
+
if spec.default is not None
|
|
227
|
+
else spec.default
|
|
228
|
+
)
|
|
229
|
+
else:
|
|
230
|
+
bound[spec.name] = spec.coerce(raw, check_exists=check_paths)
|
|
231
|
+
return bound
|
|
232
|
+
|
|
233
|
+
def run(self, supplied: dict[str, Any], ctx: Context | None = None) -> OpResult:
|
|
234
|
+
bound = self.bind(supplied)
|
|
235
|
+
result = self.func(bound, ctx or Context())
|
|
236
|
+
# Lineage is recorded here rather than inside each operation, so no
|
|
237
|
+
# operation can forget it and a new one gets it without asking.
|
|
238
|
+
_record_provenance(self.name, bound, result)
|
|
239
|
+
return result
|
|
240
|
+
|
|
241
|
+
def as_dict(self) -> dict[str, Any]:
|
|
242
|
+
return {
|
|
243
|
+
"name": self.name,
|
|
244
|
+
"title": self.title,
|
|
245
|
+
"category": self.category,
|
|
246
|
+
"summary": self.summary,
|
|
247
|
+
"notes": self.notes,
|
|
248
|
+
"destructive": self.destructive,
|
|
249
|
+
"params": [p.as_dict() for p in self.params],
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _record_provenance(name: str, params: dict[str, Any], result: OpResult) -> None:
|
|
254
|
+
"""Append to the output folder's history, never failing the run for it.
|
|
255
|
+
|
|
256
|
+
Imported lazily to keep the module graph acyclic -- provenance depends on
|
|
257
|
+
nothing but types, and registry stays the thing everything else imports.
|
|
258
|
+
"""
|
|
259
|
+
try:
|
|
260
|
+
from . import provenance
|
|
261
|
+
|
|
262
|
+
provenance.record(name, params, result, version=_version())
|
|
263
|
+
except Exception: # provenance must never break a completed operation
|
|
264
|
+
pass
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _version() -> str:
|
|
268
|
+
try:
|
|
269
|
+
from .. import __version__
|
|
270
|
+
|
|
271
|
+
return str(__version__)
|
|
272
|
+
except Exception:
|
|
273
|
+
return ""
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
_REGISTRY: dict[str, Operation] = {}
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def operation(
|
|
280
|
+
name: str,
|
|
281
|
+
*,
|
|
282
|
+
title: str,
|
|
283
|
+
category: str,
|
|
284
|
+
summary: str,
|
|
285
|
+
params: Iterable[Param] = (),
|
|
286
|
+
destructive: bool = False,
|
|
287
|
+
notes: str = "",
|
|
288
|
+
safe_when: Callable[[dict[str, Any]], bool] | None = None,
|
|
289
|
+
hidden_from_web: bool = False,
|
|
290
|
+
) -> Callable[[OpFunc], OpFunc]:
|
|
291
|
+
"""Decorator registering a function as a Data Factory operation."""
|
|
292
|
+
|
|
293
|
+
def decorate(func: OpFunc) -> OpFunc:
|
|
294
|
+
if name in _REGISTRY:
|
|
295
|
+
raise ValueError(f"duplicate operation name: {name}")
|
|
296
|
+
_REGISTRY[name] = Operation(
|
|
297
|
+
name=name,
|
|
298
|
+
title=title,
|
|
299
|
+
category=category,
|
|
300
|
+
summary=summary,
|
|
301
|
+
params=tuple(params),
|
|
302
|
+
func=func,
|
|
303
|
+
destructive=destructive,
|
|
304
|
+
notes=notes,
|
|
305
|
+
safe_when=safe_when,
|
|
306
|
+
hidden_from_web=hidden_from_web,
|
|
307
|
+
)
|
|
308
|
+
return func
|
|
309
|
+
|
|
310
|
+
return decorate
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def get(name: str) -> Operation:
|
|
314
|
+
try:
|
|
315
|
+
return _REGISTRY[name]
|
|
316
|
+
except KeyError:
|
|
317
|
+
raise KeyError(
|
|
318
|
+
f"unknown operation {name!r}. Known: {', '.join(sorted(_REGISTRY))}"
|
|
319
|
+
) from None
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def all_operations() -> list[Operation]:
|
|
323
|
+
return sorted(_REGISTRY.values(), key=lambda op: (op.category, op.name))
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def categories() -> dict[str, list[Operation]]:
|
|
327
|
+
grouped: dict[str, list[Operation]] = {}
|
|
328
|
+
for op in all_operations():
|
|
329
|
+
grouped.setdefault(op.category, []).append(op)
|
|
330
|
+
return grouped
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def schema() -> dict[str, Any]:
|
|
334
|
+
"""Machine-readable description of the whole tool.
|
|
335
|
+
|
|
336
|
+
``datafactory schema`` writes this to a file; a web front-end reads it and
|
|
337
|
+
renders a form per operation without importing any Python.
|
|
338
|
+
"""
|
|
339
|
+
return {"version": 1, "operations": [op.as_dict() for op in all_operations()]}
|