easyannotate 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. data_factory/__init__.py +82 -0
  2. data_factory/__main__.py +6 -0
  3. data_factory/annotate/__init__.py +15 -0
  4. data_factory/annotate/attach.py +398 -0
  5. data_factory/annotate/demo.py +161 -0
  6. data_factory/annotate/images.py +443 -0
  7. data_factory/annotate/labels.py +399 -0
  8. data_factory/annotate/loop.py +359 -0
  9. data_factory/annotate/projects.py +207 -0
  10. data_factory/annotate/review.py +471 -0
  11. data_factory/annotate/schema.sql +171 -0
  12. data_factory/annotate/shapes.py +488 -0
  13. data_factory/annotate/store.py +292 -0
  14. data_factory/annotate/tools/__init__.py +39 -0
  15. data_factory/annotate/tools/fake.py +170 -0
  16. data_factory/annotate/tools/install.py +254 -0
  17. data_factory/annotate/tools/local.py +389 -0
  18. data_factory/annotate/tools/postprocess.py +175 -0
  19. data_factory/annotate/tools/protocol.py +238 -0
  20. data_factory/annotate/tools/registry.py +196 -0
  21. data_factory/annotate/tools/results.py +83 -0
  22. data_factory/annotate/tools/serve.py +106 -0
  23. data_factory/cli/__init__.py +21 -0
  24. data_factory/cli/main.py +837 -0
  25. data_factory/cli/menu.py +173 -0
  26. data_factory/cli/render.py +158 -0
  27. data_factory/core/__init__.py +11 -0
  28. data_factory/core/annotations/__init__.py +5 -0
  29. data_factory/core/annotations/coco.py +394 -0
  30. data_factory/core/annotations/csvfmt.py +174 -0
  31. data_factory/core/annotations/dota.py +98 -0
  32. data_factory/core/annotations/formats.py +626 -0
  33. data_factory/core/annotations/labelme.py +153 -0
  34. data_factory/core/annotations/model.py +334 -0
  35. data_factory/core/annotations/voc.py +231 -0
  36. data_factory/core/annotations/yolo.py +283 -0
  37. data_factory/core/bktree.py +89 -0
  38. data_factory/core/fs.py +245 -0
  39. data_factory/core/imaging.py +626 -0
  40. data_factory/core/ops/__init__.py +44 -0
  41. data_factory/core/ops/analyze.py +383 -0
  42. data_factory/core/ops/annotate_export.py +326 -0
  43. data_factory/core/ops/audit.py +274 -0
  44. data_factory/core/ops/augment.py +454 -0
  45. data_factory/core/ops/collect.py +241 -0
  46. data_factory/core/ops/combine.py +582 -0
  47. data_factory/core/ops/convert.py +212 -0
  48. data_factory/core/ops/dataset.py +246 -0
  49. data_factory/core/ops/dedup.py +235 -0
  50. data_factory/core/ops/export.py +168 -0
  51. data_factory/core/ops/image.py +407 -0
  52. data_factory/core/ops/labels.py +215 -0
  53. data_factory/core/ops/pairing.py +133 -0
  54. data_factory/core/ops/rename.py +154 -0
  55. data_factory/core/ops/report.py +323 -0
  56. data_factory/core/ops/visualize.py +164 -0
  57. data_factory/core/pairs.py +144 -0
  58. data_factory/core/parallel.py +191 -0
  59. data_factory/core/provenance.py +136 -0
  60. data_factory/core/registry.py +339 -0
  61. data_factory/core/stats.py +403 -0
  62. data_factory/core/types.py +148 -0
  63. data_factory/core/writer.py +194 -0
  64. data_factory/desktop.py +207 -0
  65. data_factory/hardware.py +243 -0
  66. data_factory/pipeline.py +192 -0
  67. data_factory/py.typed +0 -0
  68. data_factory/service/__init__.py +26 -0
  69. data_factory/service/app.py +483 -0
  70. data_factory/service/binding.py +174 -0
  71. data_factory/service/deps.py +133 -0
  72. data_factory/service/jobs.py +177 -0
  73. data_factory/service/limits.py +71 -0
  74. data_factory/service/routers/__init__.py +11 -0
  75. data_factory/service/routers/annotate.py +582 -0
  76. data_factory/service/routers/local.py +464 -0
  77. data_factory/service/routers/media.py +154 -0
  78. data_factory/service/routers/predictions.py +116 -0
  79. data_factory/service/routers/promote.py +182 -0
  80. data_factory/service/routers/tools.py +308 -0
  81. data_factory/service/static/annotate/api.js +246 -0
  82. data_factory/service/static/annotate/app.js +1696 -0
  83. data_factory/service/static/annotate/assist.js +106 -0
  84. data_factory/service/static/annotate/filmstrip.js +286 -0
  85. data_factory/service/static/annotate/hit.js +146 -0
  86. data_factory/service/static/annotate/labels.js +197 -0
  87. data_factory/service/static/annotate/render.js +286 -0
  88. data_factory/service/static/annotate/scene.js +277 -0
  89. data_factory/service/static/annotate/shapelist.js +72 -0
  90. data_factory/service/static/annotate/tools/box.js +96 -0
  91. data_factory/service/static/annotate/tools/line.js +96 -0
  92. data_factory/service/static/annotate/tools/obb.js +88 -0
  93. data_factory/service/static/annotate/tools/point.js +49 -0
  94. data_factory/service/static/annotate/tools/polygon.js +110 -0
  95. data_factory/service/static/annotate/tools/select.js +227 -0
  96. data_factory/service/static/annotate/undo.js +176 -0
  97. data_factory/service/static/annotate/viewport.js +156 -0
  98. data_factory/service/static/annotate.html +390 -0
  99. data_factory/service/static/index.html +632 -0
  100. data_factory/service/workspace.py +244 -0
  101. easyannotate/__init__.py +64 -0
  102. easyannotate/__main__.py +5 -0
  103. easyannotate/project.py +181 -0
  104. easyannotate/py.typed +0 -0
  105. easyannotate-0.4.0.dist-info/METADATA +767 -0
  106. easyannotate-0.4.0.dist-info/RECORD +109 -0
  107. easyannotate-0.4.0.dist-info/WHEEL +4 -0
  108. easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
  109. easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,191 @@
1
+ """Parallel map for the decode-bound stages.
2
+
3
+ Perceptual hashing 2,783 images took 67 seconds, and essentially all of it was
4
+ JPEG decode -- one core busy, the rest idle. Decoding is embarrassingly
5
+ parallel, so this is the piece that was missing rather than a smarter
6
+ algorithm; the BK-tree already fixed the comparison cost.
7
+
8
+ Threads, not processes. OpenCV and Pillow release the GIL around decode, so
9
+ threads get real parallelism here, and they avoid pickling a few thousand image
10
+ paths and hash objects across a process boundary. A process pool would also
11
+ break the Windows entry-point rules for a library that gets imported by a web
12
+ server.
13
+
14
+ Two properties matter more than the speed:
15
+
16
+ *Order is preserved.* Dedup keeps the first image it sees, so results that
17
+ depended on which thread finished first would not be reproducible. Results come
18
+ back in input order regardless of completion order.
19
+
20
+ *Cancellation and progress still work.* Both are polled from the calling
21
+ thread as results arrive, so a web request can still abort a long run.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ import os
27
+ from collections import deque
28
+ from collections.abc import Callable, Iterator, Sequence
29
+ from concurrent.futures import ThreadPoolExecutor
30
+ from typing import Any, TypeVar
31
+
32
+ from .types import Context
33
+
34
+ T = TypeVar("T")
35
+ R = TypeVar("R")
36
+
37
+ #: Past this, threads contend on memory bandwidth rather than adding speed, and
38
+ #: a 32-core machine should not open 32 large JPEGs at once.
39
+ MAX_WORKERS = 8
40
+
41
+
42
+ def resolve_workers(requested: int) -> int:
43
+ """Turn a ``--workers`` value into an actual thread count.
44
+
45
+ 0 means "decide for me": one per core, capped, and always leaving the
46
+ machine usable.
47
+ """
48
+ if requested and requested > 0:
49
+ return max(1, min(int(requested), 32))
50
+ cores = os.cpu_count() or 2
51
+ return max(1, min(cores - 1, MAX_WORKERS))
52
+
53
+
54
+ def map_ordered(
55
+ items: Sequence[T],
56
+ work: Callable[[T], R],
57
+ *,
58
+ ctx: Context,
59
+ workers: int = 0,
60
+ label: Callable[[T], str] | None = None,
61
+ on_error: Callable[[T, Exception], None] | None = None,
62
+ ) -> Iterator[tuple[T, R | None]]:
63
+ """Apply ``work`` to every item, yielding ``(item, result)`` in input order.
64
+
65
+ Items whose work raised yield ``None`` after ``on_error`` is called, so a
66
+ single corrupt file cannot abort a scan of twenty thousand.
67
+ """
68
+ total = len(items)
69
+ if total == 0:
70
+ return
71
+
72
+ count = resolve_workers(workers)
73
+ if count == 1 or total == 1:
74
+ # Serial path: identical semantics, no thread pool overhead, and a much
75
+ # easier thing to step through in a debugger.
76
+ for position, item in enumerate(items, start=1):
77
+ ctx.check_cancelled()
78
+ ctx.emit(position, total, label(item) if label else "")
79
+ try:
80
+ yield item, work(item)
81
+ except Exception as exc:
82
+ if on_error is not None:
83
+ on_error(item, exc)
84
+ yield item, None
85
+ return
86
+
87
+ with ThreadPoolExecutor(max_workers=count) as pool:
88
+ futures = [pool.submit(work, item) for item in items]
89
+ for position, (item, future) in enumerate(zip(items, futures), start=1):
90
+ try:
91
+ ctx.check_cancelled()
92
+ except BaseException:
93
+ # Stop the queue draining into work nobody will read. Already
94
+ # running tasks finish; the rest never start.
95
+ for pending in futures[position - 1 :]:
96
+ pending.cancel()
97
+ raise
98
+ ctx.emit(position, total, label(item) if label else "")
99
+ try:
100
+ yield item, future.result()
101
+ except Exception as exc:
102
+ if on_error is not None:
103
+ on_error(item, exc)
104
+ yield item, None
105
+
106
+
107
+ def map_bounded(
108
+ items: Sequence[T],
109
+ work: Callable[[T], R],
110
+ *,
111
+ ctx: Context,
112
+ workers: int = 0,
113
+ label: Callable[[T], str] | None = None,
114
+ on_error: Callable[[T, Exception], None] | None = None,
115
+ in_flight: int = 0,
116
+ ) -> Iterator[tuple[T, R | None]]:
117
+ """:func:`map_ordered`, but with a bounded number of tasks in flight.
118
+
119
+ The difference matters when the work returns something *large*.
120
+ ``map_ordered`` submits every task up front, which is right for hashing --
121
+ a hash is 8 bytes -- and catastrophic for anything returning decoded or
122
+ encoded image data: 3,686 frames would all be resident at once.
123
+
124
+ Here at most ``in_flight`` results exist simultaneously, so memory stays
125
+ flat regardless of dataset size. Order is preserved exactly as before.
126
+ """
127
+ total = len(items)
128
+ if total == 0:
129
+ return
130
+
131
+ count = resolve_workers(workers)
132
+ if count == 1 or total == 1:
133
+ yield from map_ordered(
134
+ items, work, ctx=ctx, workers=1, label=label, on_error=on_error
135
+ )
136
+ return
137
+
138
+ limit = in_flight if in_flight > 0 else max(2, count * 2)
139
+ pending: deque[tuple[T, Any]] = deque()
140
+ position = 0
141
+
142
+ def take() -> tuple[T, R | None]:
143
+ nonlocal position
144
+ item, future = pending.popleft()
145
+ position += 1
146
+ ctx.emit(position, total, label(item) if label else "")
147
+ try:
148
+ return item, future.result()
149
+ except Exception as exc:
150
+ if on_error is not None:
151
+ on_error(item, exc)
152
+ return item, None
153
+
154
+ with ThreadPoolExecutor(max_workers=count) as pool:
155
+ try:
156
+ for item in items:
157
+ ctx.check_cancelled()
158
+ pending.append((item, pool.submit(work, item)))
159
+ if len(pending) >= limit:
160
+ yield take()
161
+ while pending:
162
+ ctx.check_cancelled()
163
+ yield take()
164
+ except BaseException:
165
+ for _, future in pending:
166
+ future.cancel()
167
+ raise
168
+
169
+
170
+ def collect(
171
+ items: Sequence[T],
172
+ work: Callable[[T], R],
173
+ *,
174
+ ctx: Context,
175
+ workers: int = 0,
176
+ label: Callable[[T], str] | None = None,
177
+ on_error: Callable[[T, Exception], None] | None = None,
178
+ ) -> list[tuple[T, R]]:
179
+ """:func:`map_ordered` with the failures dropped, as a list."""
180
+ return [
181
+ (item, value)
182
+ for item, value in map_ordered(
183
+ items, work, ctx=ctx, workers=workers, label=label, on_error=on_error
184
+ )
185
+ if value is not None
186
+ ]
187
+
188
+
189
+ WORKERS_HELP = (
190
+ "Parallel worker threads for image decoding (0 = choose automatically)."
191
+ )
@@ -0,0 +1,136 @@
1
+ """Dataset lineage: what was done to this folder, and with what settings.
2
+
3
+ Six weeks after a dataset is built, the question that matters is "what is this,
4
+ exactly" -- was it augmented, at what seed, was it resized before or after the
5
+ split, which source folders went into the merge. Without a record that question
6
+ is unanswerable, and the usual answer is to rebuild the dataset from scratch
7
+ and hope.
8
+
9
+ So every operation that writes a dataset appends one line to
10
+ ``<output>/.datafactory/history.jsonl``. It is recorded in
11
+ :meth:`Operation.run`, not by the operations themselves, which means no
12
+ operation can forget to do it and no new operation has to remember.
13
+
14
+ JSON Lines rather than a single JSON document, because appending must not
15
+ require reading and rewriting the file -- a build that dies halfway still
16
+ leaves a readable, truthful history of what it managed to do.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import json
22
+ import os
23
+ from datetime import UTC, datetime
24
+ from pathlib import Path
25
+ from typing import Any
26
+
27
+ from .types import OpResult, _jsonable
28
+
29
+ HISTORY_DIR = ".datafactory"
30
+ HISTORY_FILE = "history.jsonl"
31
+
32
+ #: Parameters naming where output goes. Whichever is present decides which
33
+ #: folder gets the history entry.
34
+ OUTPUT_KEYS = ("output", "quarantine")
35
+
36
+ #: Never copy these into a record -- they are noise, or they are large.
37
+ SKIP_KEYS = frozenset({"workers", "quality", "recursive"})
38
+
39
+
40
+ def history_path(folder: Path) -> Path:
41
+ return Path(folder) / HISTORY_DIR / HISTORY_FILE
42
+
43
+
44
+ def record(
45
+ op_name: str,
46
+ params: dict[str, Any],
47
+ result: OpResult,
48
+ *,
49
+ version: str = "",
50
+ ) -> Path | None:
51
+ """Append one entry to the output folder's history. Returns the file written.
52
+
53
+ Silent no-op when the run was a dry run, failed, or produced no output
54
+ folder -- there is nothing to describe the provenance *of*.
55
+ """
56
+ if result.dry_run or not result.ok:
57
+ return None
58
+ folder = _output_folder(params)
59
+ if folder is None or not folder.is_dir():
60
+ return None
61
+
62
+ entry = {
63
+ "at": datetime.now(UTC).isoformat(timespec="seconds"),
64
+ "op": op_name,
65
+ "version": version,
66
+ "params": _describe(params),
67
+ "counts": dict(result.counts),
68
+ "warnings": len(result.warnings),
69
+ "produced": _jsonable(result.data.get("format")) or None,
70
+ }
71
+
72
+ target = history_path(folder)
73
+ try:
74
+ target.parent.mkdir(parents=True, exist_ok=True)
75
+ with target.open("a", encoding="utf-8") as handle:
76
+ handle.write(json.dumps(entry, ensure_ascii=False) + "\n")
77
+ except OSError:
78
+ # Provenance is valuable but never worth failing a completed build for.
79
+ return None
80
+ return target
81
+
82
+
83
+ def read(folder: Path) -> list[dict[str, Any]]:
84
+ """Read a folder's history, skipping any line that got truncated."""
85
+ path = history_path(folder)
86
+ if not path.is_file():
87
+ return []
88
+ entries: list[dict[str, Any]] = []
89
+ for line in path.read_text(encoding="utf-8").splitlines():
90
+ line = line.strip()
91
+ if not line:
92
+ continue
93
+ try:
94
+ entries.append(json.loads(line))
95
+ except json.JSONDecodeError:
96
+ continue
97
+ return entries
98
+
99
+
100
+ def _output_folder(params: dict[str, Any]) -> Path | None:
101
+ for key in OUTPUT_KEYS:
102
+ value = params.get(key)
103
+ if isinstance(value, Path):
104
+ return value
105
+ return None
106
+
107
+
108
+ def _describe(params: dict[str, Any]) -> dict[str, Any]:
109
+ """The settings worth keeping, as JSON.
110
+
111
+ Defaults are kept as well as overrides: "what seed was this augmented at"
112
+ has to be answerable without also knowing what the default was in the
113
+ version that built it.
114
+ """
115
+ described: dict[str, Any] = {}
116
+ for key, value in params.items():
117
+ if key in SKIP_KEYS or value is None:
118
+ continue
119
+ if isinstance(value, Path):
120
+ described[key] = _short(value)
121
+ elif isinstance(value, (list, tuple)):
122
+ described[key] = [_short(v) if isinstance(v, Path) else v for v in value]
123
+ else:
124
+ described[key] = _jsonable(value)
125
+ return described
126
+
127
+
128
+ def _short(path: Path) -> str:
129
+ """Absolute paths, but relative to the working directory when that is
130
+ shorter -- a history full of D:/long/absolute/prefixes is unreadable."""
131
+ text = str(path)
132
+ try:
133
+ relative = os.path.relpath(path, Path.cwd())
134
+ except (ValueError, OSError):
135
+ return text
136
+ return relative if len(relative) < len(text) and not relative.startswith("..") else text
@@ -0,0 +1,339 @@
1
+ """The operation registry: one declaration, four front-ends.
2
+
3
+ Every operation states its own parameters once. From that single declaration we
4
+ generate the argparse subcommand, the interactive menu prompts, the YAML step
5
+ validator, and the JSON schema a web app turns into a form. Adding an operation
6
+ therefore means writing one file -- it appears everywhere else for free, and no
7
+ front-end has to be edited.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from collections.abc import Callable, Iterable
13
+ from dataclasses import dataclass
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ from .types import Context, OpResult
18
+
19
+ #: Parameter kinds. ``dir``/``file``/``path`` are all strings at runtime but
20
+ #: carry different intent -- a web form renders a folder picker for ``dir`` and
21
+ #: the CLI can validate existence before an operation starts.
22
+ PARAM_KINDS = (
23
+ "dir",
24
+ "file",
25
+ "path",
26
+ "str",
27
+ "int",
28
+ "float",
29
+ "bool",
30
+ "choice",
31
+ "dirs",
32
+ "strs",
33
+ "mapping",
34
+ )
35
+
36
+
37
+ class ParamError(ValueError):
38
+ """Raised when supplied parameters are missing, unknown, or ill-typed."""
39
+
40
+
41
+ @dataclass(slots=True)
42
+ class Param:
43
+ name: str
44
+ kind: str
45
+ help: str
46
+ default: Any = None
47
+ required: bool = False
48
+ choices: tuple[str, ...] | None = None
49
+ minimum: float | None = None
50
+ maximum: float | None = None
51
+ must_exist: bool = False
52
+
53
+ def __post_init__(self) -> None:
54
+ if self.kind not in PARAM_KINDS:
55
+ raise ValueError(f"{self.name}: unknown param kind {self.kind!r}")
56
+ if self.kind == "choice" and not self.choices:
57
+ raise ValueError(f"{self.name}: choice params need choices")
58
+
59
+ # -- coercion ----------------------------------------------------------
60
+ def coerce(self, raw: Any, *, check_exists: bool = True) -> Any:
61
+ """Turn a string (CLI/YAML/HTTP form) into the real Python value.
62
+
63
+ ``check_exists`` is switched off when validating a recipe up front: a
64
+ later step's input folder is legitimately created by an earlier step,
65
+ so it will not exist yet at validation time.
66
+ """
67
+ if raw is None:
68
+ return None
69
+ verify = self.must_exist and check_exists
70
+ if self.kind in ("dir", "file", "path"):
71
+ path = Path(str(raw)).expanduser()
72
+ if verify and not path.exists():
73
+ raise ParamError(f"{self.name}: path does not exist: {path}")
74
+ if verify and self.kind == "dir" and not path.is_dir():
75
+ raise ParamError(f"{self.name}: not a directory: {path}")
76
+ return path
77
+ if self.kind == "dirs":
78
+ items = raw if isinstance(raw, (list, tuple)) else _split_list(str(raw))
79
+ paths = [Path(str(i)).expanduser() for i in items]
80
+ if verify:
81
+ missing = [str(p) for p in paths if not p.is_dir()]
82
+ if missing:
83
+ raise ParamError(f"{self.name}: missing folders: {', '.join(missing)}")
84
+ return paths
85
+ if self.kind == "strs":
86
+ if isinstance(raw, (list, tuple)):
87
+ return [str(i) for i in raw]
88
+ return _split_list(str(raw))
89
+ if self.kind == "mapping":
90
+ return _coerce_mapping(self.name, raw)
91
+ if self.kind == "int":
92
+ value = int(raw)
93
+ self._check_range(value)
94
+ return value
95
+ if self.kind == "float":
96
+ number = float(raw)
97
+ self._check_range(number)
98
+ return number
99
+ if self.kind == "bool":
100
+ if isinstance(raw, bool):
101
+ return raw
102
+ text = str(raw).strip().lower()
103
+ if text in ("1", "true", "yes", "y", "on"):
104
+ return True
105
+ if text in ("0", "false", "no", "n", "off"):
106
+ return False
107
+ raise ParamError(f"{self.name}: expected a boolean, got {raw!r}")
108
+ if self.kind == "choice":
109
+ text = str(raw)
110
+ assert self.choices is not None
111
+ if text not in self.choices:
112
+ raise ParamError(
113
+ f"{self.name}: {text!r} is not one of {', '.join(self.choices)}"
114
+ )
115
+ return text
116
+ return str(raw)
117
+
118
+ def _check_range(self, value: float) -> None:
119
+ if self.minimum is not None and value < self.minimum:
120
+ raise ParamError(f"{self.name}: must be >= {self.minimum}")
121
+ if self.maximum is not None and value > self.maximum:
122
+ raise ParamError(f"{self.name}: must be <= {self.maximum}")
123
+
124
+ # -- introspection -----------------------------------------------------
125
+ def as_dict(self) -> dict[str, Any]:
126
+ out: dict[str, Any] = {
127
+ "name": self.name,
128
+ "kind": self.kind,
129
+ "help": self.help,
130
+ "required": self.required,
131
+ "default": str(self.default) if isinstance(self.default, Path) else self.default,
132
+ }
133
+ if self.choices:
134
+ out["choices"] = list(self.choices)
135
+ if self.minimum is not None:
136
+ out["minimum"] = self.minimum
137
+ if self.maximum is not None:
138
+ out["maximum"] = self.maximum
139
+ return out
140
+
141
+
142
+ def _split_list(text: str) -> list[str]:
143
+ """Split a comma- or semicolon-separated list, tolerating stray spaces."""
144
+ parts = [p.strip() for chunk in text.split(";") for p in chunk.split(",")]
145
+ return [p for p in parts if p]
146
+
147
+
148
+ def _coerce_mapping(name: str, raw: Any) -> dict[str, str]:
149
+ """Accept a dict (YAML/JSON) or an ``old=new,old2=new2`` string (CLI)."""
150
+ if isinstance(raw, dict):
151
+ return {str(k): str(v) for k, v in raw.items()}
152
+ mapping: dict[str, str] = {}
153
+ for item in _split_list(str(raw)):
154
+ if "=" not in item:
155
+ raise ParamError(f"{name}: expected old=new pairs, got {item!r}")
156
+ old, new = item.split("=", 1)
157
+ mapping[old.strip()] = new.strip()
158
+ return mapping
159
+
160
+
161
+ OpFunc = Callable[[dict[str, Any], Context], OpResult]
162
+
163
+
164
+ @dataclass(slots=True)
165
+ class Operation:
166
+ """A single unit of work, plus everything a front-end needs to drive it."""
167
+
168
+ name: str
169
+ title: str
170
+ category: str
171
+ summary: str
172
+ params: tuple[Param, ...]
173
+ func: OpFunc
174
+ destructive: bool = False
175
+ #: Longer prose shown by ``datafactory explain <op>`` and web tooltips.
176
+ notes: str = ""
177
+ #: Given bound parameters, return True when this particular run changes
178
+ #: nothing -- ``--action report`` on an otherwise destructive operation, say.
179
+ safe_when: Callable[[dict[str, Any]], bool] | None = None
180
+ #: Keep this operation out of the web form list while leaving it in
181
+ #: ``datafactory list`` and in recipes. For operations whose real inputs
182
+ #: are not a folder the browser can name -- an annotation project, say --
183
+ #: so rendering a form for them would only invite a refused request.
184
+ hidden_from_web: bool = False
185
+
186
+ def destructive_for(self, params: dict[str, Any]) -> bool:
187
+ """Whether *this* invocation, with these parameters, writes anything."""
188
+ if not self.destructive:
189
+ return False
190
+ if self.safe_when is not None and self.safe_when(params):
191
+ return False
192
+ return True
193
+
194
+ def param(self, name: str) -> Param:
195
+ for candidate in self.params:
196
+ if candidate.name == name:
197
+ return candidate
198
+ raise KeyError(name)
199
+
200
+ def bind(self, supplied: dict[str, Any], *, check_paths: bool = True) -> dict[str, Any]:
201
+ """Validate supplied values, apply defaults, and coerce types.
202
+
203
+ Every front-end funnels through here, so a YAML recipe and an HTTP POST
204
+ get byte-identical validation to a typed command line.
205
+
206
+ ``check_paths=False`` validates everything except whether paths exist --
207
+ what a recipe needs at load time, since step 7's input is step 6's output.
208
+ """
209
+ known = {p.name for p in self.params}
210
+ unknown = set(supplied) - known
211
+ if unknown:
212
+ raise ParamError(
213
+ f"{self.name}: unknown parameter(s): {', '.join(sorted(unknown))}. "
214
+ f"Valid: {', '.join(sorted(known))}"
215
+ )
216
+ bound: dict[str, Any] = {}
217
+ for spec in self.params:
218
+ raw = supplied.get(spec.name)
219
+ if raw is None or raw == "":
220
+ if spec.required:
221
+ raise ParamError(
222
+ f"{self.name}: missing required parameter {spec.name!r} ({spec.help})"
223
+ )
224
+ bound[spec.name] = (
225
+ spec.coerce(spec.default, check_exists=check_paths)
226
+ if spec.default is not None
227
+ else spec.default
228
+ )
229
+ else:
230
+ bound[spec.name] = spec.coerce(raw, check_exists=check_paths)
231
+ return bound
232
+
233
+ def run(self, supplied: dict[str, Any], ctx: Context | None = None) -> OpResult:
234
+ bound = self.bind(supplied)
235
+ result = self.func(bound, ctx or Context())
236
+ # Lineage is recorded here rather than inside each operation, so no
237
+ # operation can forget it and a new one gets it without asking.
238
+ _record_provenance(self.name, bound, result)
239
+ return result
240
+
241
+ def as_dict(self) -> dict[str, Any]:
242
+ return {
243
+ "name": self.name,
244
+ "title": self.title,
245
+ "category": self.category,
246
+ "summary": self.summary,
247
+ "notes": self.notes,
248
+ "destructive": self.destructive,
249
+ "params": [p.as_dict() for p in self.params],
250
+ }
251
+
252
+
253
+ def _record_provenance(name: str, params: dict[str, Any], result: OpResult) -> None:
254
+ """Append to the output folder's history, never failing the run for it.
255
+
256
+ Imported lazily to keep the module graph acyclic -- provenance depends on
257
+ nothing but types, and registry stays the thing everything else imports.
258
+ """
259
+ try:
260
+ from . import provenance
261
+
262
+ provenance.record(name, params, result, version=_version())
263
+ except Exception: # provenance must never break a completed operation
264
+ pass
265
+
266
+
267
+ def _version() -> str:
268
+ try:
269
+ from .. import __version__
270
+
271
+ return str(__version__)
272
+ except Exception:
273
+ return ""
274
+
275
+
276
+ _REGISTRY: dict[str, Operation] = {}
277
+
278
+
279
+ def operation(
280
+ name: str,
281
+ *,
282
+ title: str,
283
+ category: str,
284
+ summary: str,
285
+ params: Iterable[Param] = (),
286
+ destructive: bool = False,
287
+ notes: str = "",
288
+ safe_when: Callable[[dict[str, Any]], bool] | None = None,
289
+ hidden_from_web: bool = False,
290
+ ) -> Callable[[OpFunc], OpFunc]:
291
+ """Decorator registering a function as a Data Factory operation."""
292
+
293
+ def decorate(func: OpFunc) -> OpFunc:
294
+ if name in _REGISTRY:
295
+ raise ValueError(f"duplicate operation name: {name}")
296
+ _REGISTRY[name] = Operation(
297
+ name=name,
298
+ title=title,
299
+ category=category,
300
+ summary=summary,
301
+ params=tuple(params),
302
+ func=func,
303
+ destructive=destructive,
304
+ notes=notes,
305
+ safe_when=safe_when,
306
+ hidden_from_web=hidden_from_web,
307
+ )
308
+ return func
309
+
310
+ return decorate
311
+
312
+
313
+ def get(name: str) -> Operation:
314
+ try:
315
+ return _REGISTRY[name]
316
+ except KeyError:
317
+ raise KeyError(
318
+ f"unknown operation {name!r}. Known: {', '.join(sorted(_REGISTRY))}"
319
+ ) from None
320
+
321
+
322
+ def all_operations() -> list[Operation]:
323
+ return sorted(_REGISTRY.values(), key=lambda op: (op.category, op.name))
324
+
325
+
326
+ def categories() -> dict[str, list[Operation]]:
327
+ grouped: dict[str, list[Operation]] = {}
328
+ for op in all_operations():
329
+ grouped.setdefault(op.category, []).append(op)
330
+ return grouped
331
+
332
+
333
+ def schema() -> dict[str, Any]:
334
+ """Machine-readable description of the whole tool.
335
+
336
+ ``datafactory schema`` writes this to a file; a web front-end reads it and
337
+ renders a form per operation without importing any Python.
338
+ """
339
+ return {"version": 1, "operations": [op.as_dict() for op in all_operations()]}