easyannotate 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. data_factory/__init__.py +82 -0
  2. data_factory/__main__.py +6 -0
  3. data_factory/annotate/__init__.py +15 -0
  4. data_factory/annotate/attach.py +398 -0
  5. data_factory/annotate/demo.py +161 -0
  6. data_factory/annotate/images.py +443 -0
  7. data_factory/annotate/labels.py +399 -0
  8. data_factory/annotate/loop.py +359 -0
  9. data_factory/annotate/projects.py +207 -0
  10. data_factory/annotate/review.py +471 -0
  11. data_factory/annotate/schema.sql +171 -0
  12. data_factory/annotate/shapes.py +488 -0
  13. data_factory/annotate/store.py +292 -0
  14. data_factory/annotate/tools/__init__.py +39 -0
  15. data_factory/annotate/tools/fake.py +170 -0
  16. data_factory/annotate/tools/install.py +254 -0
  17. data_factory/annotate/tools/local.py +389 -0
  18. data_factory/annotate/tools/postprocess.py +175 -0
  19. data_factory/annotate/tools/protocol.py +238 -0
  20. data_factory/annotate/tools/registry.py +196 -0
  21. data_factory/annotate/tools/results.py +83 -0
  22. data_factory/annotate/tools/serve.py +106 -0
  23. data_factory/cli/__init__.py +21 -0
  24. data_factory/cli/main.py +837 -0
  25. data_factory/cli/menu.py +173 -0
  26. data_factory/cli/render.py +158 -0
  27. data_factory/core/__init__.py +11 -0
  28. data_factory/core/annotations/__init__.py +5 -0
  29. data_factory/core/annotations/coco.py +394 -0
  30. data_factory/core/annotations/csvfmt.py +174 -0
  31. data_factory/core/annotations/dota.py +98 -0
  32. data_factory/core/annotations/formats.py +626 -0
  33. data_factory/core/annotations/labelme.py +153 -0
  34. data_factory/core/annotations/model.py +334 -0
  35. data_factory/core/annotations/voc.py +231 -0
  36. data_factory/core/annotations/yolo.py +283 -0
  37. data_factory/core/bktree.py +89 -0
  38. data_factory/core/fs.py +245 -0
  39. data_factory/core/imaging.py +626 -0
  40. data_factory/core/ops/__init__.py +44 -0
  41. data_factory/core/ops/analyze.py +383 -0
  42. data_factory/core/ops/annotate_export.py +326 -0
  43. data_factory/core/ops/audit.py +274 -0
  44. data_factory/core/ops/augment.py +454 -0
  45. data_factory/core/ops/collect.py +241 -0
  46. data_factory/core/ops/combine.py +582 -0
  47. data_factory/core/ops/convert.py +212 -0
  48. data_factory/core/ops/dataset.py +246 -0
  49. data_factory/core/ops/dedup.py +235 -0
  50. data_factory/core/ops/export.py +168 -0
  51. data_factory/core/ops/image.py +407 -0
  52. data_factory/core/ops/labels.py +215 -0
  53. data_factory/core/ops/pairing.py +133 -0
  54. data_factory/core/ops/rename.py +154 -0
  55. data_factory/core/ops/report.py +323 -0
  56. data_factory/core/ops/visualize.py +164 -0
  57. data_factory/core/pairs.py +144 -0
  58. data_factory/core/parallel.py +191 -0
  59. data_factory/core/provenance.py +136 -0
  60. data_factory/core/registry.py +339 -0
  61. data_factory/core/stats.py +403 -0
  62. data_factory/core/types.py +148 -0
  63. data_factory/core/writer.py +194 -0
  64. data_factory/desktop.py +207 -0
  65. data_factory/hardware.py +243 -0
  66. data_factory/pipeline.py +192 -0
  67. data_factory/py.typed +0 -0
  68. data_factory/service/__init__.py +26 -0
  69. data_factory/service/app.py +483 -0
  70. data_factory/service/binding.py +174 -0
  71. data_factory/service/deps.py +133 -0
  72. data_factory/service/jobs.py +177 -0
  73. data_factory/service/limits.py +71 -0
  74. data_factory/service/routers/__init__.py +11 -0
  75. data_factory/service/routers/annotate.py +582 -0
  76. data_factory/service/routers/local.py +464 -0
  77. data_factory/service/routers/media.py +154 -0
  78. data_factory/service/routers/predictions.py +116 -0
  79. data_factory/service/routers/promote.py +182 -0
  80. data_factory/service/routers/tools.py +308 -0
  81. data_factory/service/static/annotate/api.js +246 -0
  82. data_factory/service/static/annotate/app.js +1696 -0
  83. data_factory/service/static/annotate/assist.js +106 -0
  84. data_factory/service/static/annotate/filmstrip.js +286 -0
  85. data_factory/service/static/annotate/hit.js +146 -0
  86. data_factory/service/static/annotate/labels.js +197 -0
  87. data_factory/service/static/annotate/render.js +286 -0
  88. data_factory/service/static/annotate/scene.js +277 -0
  89. data_factory/service/static/annotate/shapelist.js +72 -0
  90. data_factory/service/static/annotate/tools/box.js +96 -0
  91. data_factory/service/static/annotate/tools/line.js +96 -0
  92. data_factory/service/static/annotate/tools/obb.js +88 -0
  93. data_factory/service/static/annotate/tools/point.js +49 -0
  94. data_factory/service/static/annotate/tools/polygon.js +110 -0
  95. data_factory/service/static/annotate/tools/select.js +227 -0
  96. data_factory/service/static/annotate/undo.js +176 -0
  97. data_factory/service/static/annotate/viewport.js +156 -0
  98. data_factory/service/static/annotate.html +390 -0
  99. data_factory/service/static/index.html +632 -0
  100. data_factory/service/workspace.py +244 -0
  101. easyannotate/__init__.py +64 -0
  102. easyannotate/__main__.py +5 -0
  103. easyannotate/project.py +181 -0
  104. easyannotate/py.typed +0 -0
  105. easyannotate-0.4.0.dist-info/METADATA +767 -0
  106. easyannotate-0.4.0.dist-info/RECORD +109 -0
  107. easyannotate-0.4.0.dist-info/WHEEL +4 -0
  108. easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
  109. easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,254 @@
1
+ """Installing the model tools: one environment, chosen for this machine.
2
+
3
+ Shared by ``easyannotate tools install`` and the desktop/local web app's
4
+ "Install AI tools" button. Nothing heavy is downloaded until someone asks.
5
+
6
+ The environment lives at ``~/.datafactory/tools`` (``DATAFACTORY_TOOLS_HOME``),
7
+ apart from the editor's: torch's version matrix is resolved there alone, and
8
+ a broken install there breaks nothing but the AI buttons.
9
+
10
+ Where the pieces come from, in order of preference:
11
+
12
+ * **uv** -- bundled inside the desktop app, else on PATH. It creates the
13
+ environment and fetches a Python for it, so a frozen app with no Python of
14
+ its own can still install. Without uv, a pip install falls back to venv+pip.
15
+ * **easyannotate + easyannotate-tools** -- wheels bundled with the desktop app
16
+ (``EASYANNOTATE_WHEELS`` or the app's ``wheels/`` folder), else a source
17
+ checkout (``--source``), else PyPI.
18
+ * **PyTorch** -- the build ``hardware.plan()`` picked: the CUDA version this
19
+ machine's *driver* supports, Metal on Apple silicon, or CPU.
20
+
21
+ A ``profile.json`` beside the environment records the device and model sizes,
22
+ and the sidecar reads it, so the same code runs SAM tiny on a laptop CPU and
23
+ SAM base on a 12 GB card.
24
+ """
25
+
26
+ from __future__ import annotations
27
+
28
+ import json
29
+ import os
30
+ import shutil
31
+ import subprocess
32
+ import sys
33
+ from collections.abc import Callable
34
+ from pathlib import Path
35
+ from typing import Any
36
+
37
+ from ... import hardware
38
+ from .local import child_environment, python_for_tools
39
+
40
+ PROFILE = "profile.json"
41
+ PYTHON_VERSION = "3.12"
42
+
43
+ #: Run in the new environment. Real work on the device, not
44
+ #: torch.cuda.is_available(): measured on a 529 driver with the cu126 build,
45
+ #: that said True while every allocation failed ("device busy or unavailable").
46
+ _PROBE = '''
47
+ import json, sys
48
+ out = {"device": "cpu", "error": ""}
49
+ import torch
50
+ out["torch"] = torch.__version__
51
+ def works(device):
52
+ x = torch.randn(128, 128, device=device)
53
+ float((x @ x).sum())
54
+ try:
55
+ if torch.cuda.is_available():
56
+ works("cuda:%d" % int(sys.argv[1]))
57
+ out["device"] = "cuda"
58
+ elif getattr(torch.backends, "mps", None) and torch.backends.mps.is_available():
59
+ works("mps")
60
+ out["device"] = "mps"
61
+ except Exception as exc:
62
+ out["error"] = (type(exc).__name__ + ": " + str(exc)).splitlines()[0][:300]
63
+ print("PROBE " + json.dumps(out))
64
+ '''
65
+
66
+
67
+ class InstallError(RuntimeError):
68
+ pass
69
+
70
+
71
+ def tools_home() -> Path:
72
+ return Path(os.environ.get("DATAFACTORY_TOOLS_HOME") or Path.home() / ".datafactory" / "tools")
73
+
74
+
75
+ def tools_python(home: Path | None = None) -> Path:
76
+ home = home or tools_home()
77
+ return home / ("Scripts/python.exe" if sys.platform == "win32" else "bin/python")
78
+
79
+
80
+ def read_profile(home: Path | None = None) -> dict[str, Any]:
81
+ path = (home or tools_home()) / PROFILE
82
+ try:
83
+ return json.loads(path.read_text(encoding="utf-8"))
84
+ except (OSError, ValueError):
85
+ return {}
86
+
87
+
88
+ def status() -> dict[str, Any]:
89
+ """Installed or not, and what this machine would get."""
90
+ hw = hardware.detect()
91
+ chosen = hardware.plan(hw)
92
+ profile = read_profile()
93
+ return {
94
+ "installed": tools_python().is_file() and bool(profile),
95
+ "profile": profile,
96
+ "hardware": hw.as_dict(),
97
+ "plan": chosen.as_dict(),
98
+ "uv": bool(find_uv()),
99
+ }
100
+
101
+
102
+ def find_uv() -> str | None:
103
+ bundled = Path(getattr(sys, "_MEIPASS", "")) / "bin" / ("uv.exe" if sys.platform == "win32" else "uv")
104
+ if getattr(sys, "_MEIPASS", None) and bundled.is_file():
105
+ return str(bundled)
106
+ return shutil.which("uv")
107
+
108
+
109
+ def bundled_wheels() -> list[Path]:
110
+ folders = [os.environ.get("EASYANNOTATE_WHEELS", "")]
111
+ if getattr(sys, "_MEIPASS", None):
112
+ folders.append(str(Path(sys._MEIPASS) / "wheels")) # noqa: SLF001
113
+ for folder in folders:
114
+ if folder and Path(folder).is_dir():
115
+ found = sorted(Path(folder).glob("*.whl"))
116
+ if found:
117
+ return found
118
+ return []
119
+
120
+
121
+ def install(
122
+ *,
123
+ source: Path | None = None,
124
+ device: str | None = None,
125
+ on_line: Callable[[str], None] = print,
126
+ ) -> dict[str, Any]:
127
+ """Create or update the tools environment. Returns the written profile.
128
+
129
+ ``device`` forces "cuda", "mps" or "cpu" instead of the detected choice.
130
+ """
131
+ hw = hardware.detect()
132
+ chosen = hardware.plan(hw)
133
+ if device == "cpu" and chosen.device != "cpu":
134
+ chosen = hardware.plan(hardware.Hardware(os=hw.os, cpu_count=hw.cpu_count, ram_gb=hw.ram_gb))
135
+ elif device in ("cuda", "mps") and chosen.device != device:
136
+ raise InstallError(f"asked for {device}, but this machine looks like: {chosen.summary}")
137
+ for warning in chosen.warnings:
138
+ on_line(f"note: {warning}")
139
+ on_line(f"plan: {chosen.summary} (about {chosen.download_gb:g} GB)")
140
+
141
+ home = tools_home()
142
+ python = tools_python(home)
143
+ uv = find_uv()
144
+ # Gone until this install has verified a working PyTorch: a half-finished
145
+ # install must not still read as "installed" with the old device.
146
+ (home / PROFILE).unlink(missing_ok=True)
147
+
148
+ def run(command: list[str]) -> None:
149
+ on_line("$ " + " ".join(str(part) for part in command))
150
+ try:
151
+ process = subprocess.Popen(
152
+ [str(part) for part in command],
153
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
154
+ encoding="utf-8", errors="replace", env=child_environment(),
155
+ )
156
+ except OSError as exc:
157
+ raise InstallError(f"could not start {command[0]}: {exc}") from None
158
+ assert process.stdout is not None
159
+ try:
160
+ for line in process.stdout:
161
+ line = line.rstrip()
162
+ if line:
163
+ on_line(line)
164
+ code = process.wait()
165
+ finally:
166
+ # Cancelled, or the server stopping: never leave pip running on.
167
+ if process.poll() is None:
168
+ process.kill()
169
+ process.wait()
170
+ if code != 0:
171
+ raise InstallError(f"failed: {' '.join(str(part) for part in command[:4])} ...")
172
+
173
+ if not python.is_file():
174
+ on_line(f"creating {home}")
175
+ if uv:
176
+ run([uv, "venv", str(home), "--python", PYTHON_VERSION])
177
+ elif not getattr(sys, "frozen", False):
178
+ run([sys.executable, "-m", "venv", str(home)])
179
+ else:
180
+ raise InstallError("this build has no uv to create the environment with")
181
+ pip = [uv, "pip", "install", "--python", str(python)] if uv else [str(python), "-m", "pip", "install"]
182
+
183
+ if chosen.torch_index:
184
+ on_line("installing PyTorch -- the large part of the download")
185
+ # Reinstall, not install: a PyTorch already there -- the CPU build,
186
+ # say -- "satisfies" plain `torch`, so asking for the CUDA build would
187
+ # quietly change nothing. Measured: that is exactly what happened.
188
+ force = (
189
+ ["--reinstall-package", "torch", "--reinstall-package", "torchvision"]
190
+ if uv else ["--force-reinstall"]
191
+ )
192
+ packages = list(chosen.torch_packages) or ["torch", "torchvision"]
193
+ run(pip + [*packages, "--index-url", chosen.torch_index, *force])
194
+
195
+ wheels = bundled_wheels()
196
+ if wheels:
197
+ packages = [str(w) for w in wheels]
198
+ elif source is not None:
199
+ packages = ["-e", str(source), "-e", str(Path(source) / "tools")]
200
+ else:
201
+ # Pinned to this version: the two are released together, and an
202
+ # unpinned name is whatever PyPI says it is today.
203
+ from ... import __version__
204
+
205
+ packages = [f"easyannotate-tools=={__version__}"]
206
+ on_line("installing the EasyAnnotate tools and Ultralytics")
207
+ run(pip + packages)
208
+
209
+ profile = {
210
+ "device": chosen.device,
211
+ "sam": chosen.sam,
212
+ "world": chosen.world,
213
+ "summary": chosen.summary,
214
+ "gpu_index": chosen.gpu_index,
215
+ "hardware": hw.as_dict(),
216
+ }
217
+ on_line("checking that PyTorch runs on this machine")
218
+ try:
219
+ probe = subprocess.run(
220
+ [str(python), "-I", "-c", _PROBE, str(chosen.gpu_index)],
221
+ capture_output=True, text=True, encoding="utf-8", errors="replace",
222
+ timeout=600, env=child_environment(),
223
+ )
224
+ except subprocess.TimeoutExpired:
225
+ raise InstallError("PyTorch took over 10 minutes to start -- try again, or `tools install --cpu`") from None
226
+ # rfind, not startswith: a library's output without a newline can come
227
+ # first on the same line (see serve._private_stdout).
228
+ output = probe.stdout or ""
229
+ start = output.rfind("PROBE {")
230
+ if probe.returncode != 0 or start < 0:
231
+ tail = " ".join((probe.stderr or "").strip().splitlines()[-2:])
232
+ raise InstallError(f"PyTorch does not start in {home}: {tail or 'no output'}")
233
+ try:
234
+ result = json.loads(output[start + len("PROBE "):].splitlines()[0])
235
+ except ValueError:
236
+ raise InstallError(f"PyTorch's self-check in {home} printed something unreadable") from None
237
+ seen = result.get("device") or "cpu"
238
+ profile["verified_device"] = seen
239
+ profile["torch"] = result.get("torch", "")
240
+ if chosen.device != "cpu" and seen != chosen.device:
241
+ why = result.get("error") or f"PyTorch sees {seen}"
242
+ on_line(f"warning: the {chosen.device} build does not run here ({why}) -- the tools will use the CPU")
243
+ # Then use the sizes a CPU can carry, and say what really happened.
244
+ cpu = hardware.plan(hardware.Hardware(os=hw.os, cpu_count=hw.cpu_count, ram_gb=hw.ram_gb))
245
+ profile.update({
246
+ "device": "cpu", "sam": cpu.sam, "world": cpu.world,
247
+ "summary": f"{cpu.summary} (the GPU build did not load -- see the log)",
248
+ })
249
+ (home / PROFILE).write_text(json.dumps(profile, indent=2), encoding="utf-8")
250
+ on_line(f"done: {profile['summary']} (PyTorch sees: {seen})")
251
+ return profile
252
+
253
+
254
+ __all__ = ["InstallError", "install", "python_for_tools", "read_profile", "status", "tools_home"]
@@ -0,0 +1,389 @@
1
+ """A heavy tool running as a child process, spoken to over stdio.
2
+
3
+ The legacy SAM script found this boundary by accident -- a ``multiprocessing``
4
+ worker holding two predictors -- and it was the best idea in either annotation
5
+ script. This formalises it, and fixes the three things that made the original
6
+ fragile:
7
+
8
+ * **A blocking read inside the UI callback.** Here a background thread owns the
9
+ pipe and dispatches replies to futures, so no handler ever blocks on a raw
10
+ read, and every wait has a timeout.
11
+ * **An untagged queue.** The original had one output queue and no correlation,
12
+ so a stale reply could be consumed by the wrong caller and a real result
13
+ silently dropped. Every message here carries an id and is matched on it.
14
+ * **An exception path that never replied.** The original caught, printed, and
15
+ put nothing back, so the UI waited forever. Here the child always answers and
16
+ the parent additionally fails every outstanding call if the child dies.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import os
22
+ import subprocess
23
+ import sys
24
+ import threading
25
+ import time
26
+ from concurrent.futures import Future
27
+ from pathlib import Path
28
+ from typing import Any
29
+
30
+ from .protocol import (
31
+ DEFAULT_TIMEOUTS,
32
+ ToolError,
33
+ ToolHealth,
34
+ ToolSpec,
35
+ ToolUnavailable,
36
+ decode_message,
37
+ encode_message,
38
+ raise_for,
39
+ request,
40
+ )
41
+
42
+ #: Restarts before we stop trying. A tool that dies three times is broken, and
43
+ #: retrying forever would hide that while burning CPU.
44
+ MAX_RESTARTS = 3
45
+ RESTART_BACKOFF = (0.5, 2.0, 5.0)
46
+
47
+ #: Keep the tail of the child's stderr so a failure can say *why* rather than
48
+ #: just "it died" -- the difference between a fixable report and a shrug.
49
+ STDERR_LINES = 40
50
+
51
+
52
+ class SidecarClient:
53
+ """Owns one child process and turns its stdio into method calls."""
54
+
55
+ def __init__(
56
+ self,
57
+ argv: list[str],
58
+ *,
59
+ cwd: Path | None = None,
60
+ env: dict[str, str] | None = None,
61
+ timeouts: dict[str, float] | None = None,
62
+ ) -> None:
63
+ self.argv = list(argv)
64
+ self.cwd = cwd
65
+ self.env = env
66
+ self.timeouts = {**DEFAULT_TIMEOUTS, **(timeouts or {})}
67
+
68
+ self._lock = threading.Lock()
69
+ self._process: subprocess.Popen[bytes] | None = None
70
+ self._reader: threading.Thread | None = None
71
+ self._stderr_reader: threading.Thread | None = None
72
+ self._pending: dict[int, Future] = {}
73
+ self._stderr: list[str] = []
74
+ self._next_id = 0
75
+ self._restarts = 0
76
+ self._closed = False
77
+ self._last_error = ""
78
+
79
+ # -- lifecycle ---------------------------------------------------------
80
+ def start(self) -> None:
81
+ """Spawn the child. Raises ToolUnavailable if it will not start."""
82
+ with self._lock:
83
+ if self._closed:
84
+ raise ToolUnavailable("this client has been closed")
85
+ if self._process is not None and self._process.poll() is None:
86
+ return
87
+ self._spawn_locked()
88
+
89
+ def _spawn_locked(self) -> None:
90
+ environment = {**child_environment(), **(self.env or {})}
91
+ # Unbuffered, or the child's replies sit in a pipe buffer and every
92
+ # call looks like a timeout.
93
+ environment["PYTHONUNBUFFERED"] = "1"
94
+ try:
95
+ self._process = subprocess.Popen(
96
+ self.argv,
97
+ stdin=subprocess.PIPE,
98
+ stdout=subprocess.PIPE,
99
+ stderr=subprocess.PIPE,
100
+ cwd=str(self.cwd) if self.cwd else None,
101
+ env=environment,
102
+ bufsize=0,
103
+ )
104
+ except (OSError, ValueError) as exc:
105
+ self._last_error = str(exc)
106
+ raise ToolUnavailable(f"cannot start {self.argv[0]!r}: {exc}") from None
107
+
108
+ self._stderr = []
109
+ self._reader = threading.Thread(target=self._read_stdout, args=(self._process,),
110
+ daemon=True, name="dfx-tool-out")
111
+ self._stderr_reader = threading.Thread(target=self._read_stderr, args=(self._process,),
112
+ daemon=True, name="dfx-tool-err")
113
+ self._reader.start()
114
+ self._stderr_reader.start()
115
+
116
+ def close(self) -> None:
117
+ with self._lock:
118
+ self._closed = True
119
+ process = self._process
120
+ self._process = None
121
+ if process is None:
122
+ return
123
+ try:
124
+ if process.stdin:
125
+ process.stdin.close()
126
+ process.wait(timeout=2)
127
+ except Exception:
128
+ process.kill()
129
+ finally:
130
+ self._fail_pending(ToolUnavailable("the tool was shut down"))
131
+
132
+ @property
133
+ def alive(self) -> bool:
134
+ process = self._process
135
+ return process is not None and process.poll() is None
136
+
137
+ # -- calling -----------------------------------------------------------
138
+ def call(self, method: str, params: dict[str, Any] | None = None,
139
+ *, timeout: float | None = None) -> Any:
140
+ """Send one request and wait for its reply.
141
+
142
+ Restarts the child once per call if it has died, so a crashed tool
143
+ recovers on the next click rather than staying broken until a restart.
144
+ """
145
+ if self._closed:
146
+ raise ToolUnavailable("this client has been closed")
147
+ wait = timeout if timeout is not None else self.timeouts.get(method, 15.0)
148
+
149
+ # Two attempts, because the child can die *between* the liveness check
150
+ # and the write -- a crash on the previous call is reaped
151
+ # asynchronously, so `poll()` can still say None for a moment while the
152
+ # pipe is already gone. Without the retry the caller sees a confusing
153
+ # "[Errno 22] Invalid argument" for a tool that is perfectly able to
154
+ # restart and serve the request.
155
+ last: Exception | None = None
156
+ for _ in range(2):
157
+ if not self.alive:
158
+ self._restart() # raises once restarts are exhausted
159
+
160
+ message_id, future, payload = self._enqueue(method, params)
161
+ try:
162
+ self._write(payload)
163
+ except ToolUnavailable as exc:
164
+ self._drop(message_id)
165
+ last = exc
166
+ self._kill_child() # force a fresh child on the next pass
167
+ continue
168
+
169
+ try:
170
+ reply = future.result(timeout=wait)
171
+ except TimeoutError:
172
+ self._drop(message_id)
173
+ # A tool that misses its deadline is not trustworthy
174
+ # afterwards: whatever it is still doing is unknown, and a late
175
+ # reply would arrive against an id nobody is waiting for.
176
+ # Restarting is the honest recovery -- except for health,
177
+ # which is only a question: a busy sidecar is not a broken one,
178
+ # and killing it on a status probe threw away loaded models.
179
+ if method != "health":
180
+ self._kill_child()
181
+ # The sidecar's own last words are the only clue there is.
182
+ tail = self._stderr_tail()
183
+ raise ToolError(
184
+ f"{method} took longer than {wait:g}s" + (f" -- sidecar said: {tail}" if tail else ""),
185
+ code="timeout",
186
+ ) from None
187
+ # A child that answered is working: its restart budget is earned
188
+ # back, so three crashes over a day do not disable the tools.
189
+ self._restarts = 0
190
+ return raise_for(reply)
191
+
192
+ raise last or ToolUnavailable("the tool could not be reached")
193
+
194
+ def _enqueue(self, method: str, params: dict[str, Any] | None):
195
+ with self._lock:
196
+ self._next_id += 1
197
+ message_id = self._next_id
198
+ future: Future = Future()
199
+ self._pending[message_id] = future
200
+ return message_id, future, request(message_id, method, params)
201
+
202
+ def _drop(self, message_id: int) -> None:
203
+ with self._lock:
204
+ self._pending.pop(message_id, None)
205
+
206
+ def _write(self, payload: dict[str, Any]) -> None:
207
+ process = self._process
208
+ if process is None or process.stdin is None or process.poll() is not None:
209
+ raise ToolUnavailable("the tool is not running")
210
+ try:
211
+ process.stdin.write(encode_message(payload))
212
+ process.stdin.flush()
213
+ except (BrokenPipeError, OSError) as exc:
214
+ raise ToolUnavailable(f"lost the tool while writing: {exc}") from None
215
+
216
+ # -- reading -----------------------------------------------------------
217
+ def _read_stdout(self, process: subprocess.Popen[bytes]) -> None:
218
+ stream = process.stdout
219
+ assert stream is not None
220
+ try:
221
+ for line in stream:
222
+ if not line.strip():
223
+ continue
224
+ try:
225
+ payload = decode_message(line)
226
+ except ToolError:
227
+ # Junk on stdout is a tool that printed instead of
228
+ # replying -- often a progress bar with no newline, with
229
+ # the real reply glued to its end. Salvage that; skip
230
+ # the rest rather than killing the session.
231
+ payload = _salvage(line)
232
+ if payload is None:
233
+ continue
234
+ try:
235
+ self._resolve(payload)
236
+ except Exception: # noqa: BLE001 -- one bad frame must not stop the reader
237
+ continue
238
+ except (ValueError, OSError):
239
+ pass
240
+ finally:
241
+ # The child is gone. Nothing else will ever answer the calls still
242
+ # waiting, so fail them now instead of letting each hit its
243
+ # timeout one after another.
244
+ self._fail_pending(
245
+ ToolUnavailable(f"the tool exited{self._stderr_tail()}")
246
+ )
247
+
248
+ def _read_stderr(self, process: subprocess.Popen[bytes]) -> None:
249
+ stream = process.stderr
250
+ assert stream is not None
251
+ try:
252
+ for line in stream:
253
+ text = line.decode("utf-8", errors="replace").rstrip()
254
+ if not text:
255
+ continue
256
+ self._stderr.append(text)
257
+ del self._stderr[:-STDERR_LINES]
258
+ except (ValueError, OSError):
259
+ pass
260
+
261
+ def _resolve(self, payload: dict[str, Any]) -> None:
262
+ message_id = payload.get("id")
263
+ if not isinstance(message_id, int):
264
+ return
265
+ with self._lock:
266
+ future = self._pending.pop(message_id, None)
267
+ # An unknown id is a late reply to something already abandoned. Drop
268
+ # it: matching by id is exactly what stops it poisoning the next call.
269
+ if future is not None and not future.done():
270
+ future.set_result(payload)
271
+
272
+ def _fail_pending(self, error: Exception) -> None:
273
+ with self._lock:
274
+ pending, self._pending = self._pending, {}
275
+ for future in pending.values():
276
+ if not future.done():
277
+ future.set_exception(error)
278
+
279
+ def _stderr_tail(self) -> str:
280
+ if not self._stderr:
281
+ return ""
282
+ return ": " + " / ".join(self._stderr[-3:])
283
+
284
+ def _kill_child(self) -> None:
285
+ with self._lock:
286
+ process, self._process = self._process, None
287
+ if process is not None:
288
+ try:
289
+ process.kill()
290
+ except Exception:
291
+ pass
292
+
293
+ def _restart(self) -> None:
294
+ if self._restarts >= MAX_RESTARTS:
295
+ raise ToolUnavailable(
296
+ f"the tool has died {self._restarts} times and will not be restarted"
297
+ f"{self._stderr_tail()}"
298
+ )
299
+ delay = RESTART_BACKOFF[min(self._restarts, len(RESTART_BACKOFF) - 1)]
300
+ self._restarts += 1
301
+ if self._restarts > 1:
302
+ time.sleep(delay)
303
+ with self._lock:
304
+ self._spawn_locked()
305
+
306
+
307
+ class LocalTool:
308
+ """A :class:`Tool` backed by a sidecar process."""
309
+
310
+ def __init__(self, argv: list[str], **kwargs: Any) -> None:
311
+ self.client = SidecarClient(argv, **kwargs)
312
+ self._specs: list[ToolSpec] | None = None
313
+
314
+ def describe(self) -> list[ToolSpec]:
315
+ if self._specs is None:
316
+ raw = self.client.call("describe")
317
+ self._specs = [ToolSpec.from_dict(item) for item in (raw or [])]
318
+ return list(self._specs)
319
+
320
+ def health(self, name: str = "") -> ToolHealth:
321
+ try:
322
+ return ToolHealth.from_dict(self.client.call("health", {"tool": name}) or {})
323
+ except ToolError as exc:
324
+ # Health must never raise: it is what the UI calls to decide
325
+ # whether to show a button at all.
326
+ return ToolHealth("unavailable", str(exc))
327
+
328
+ def prepare(self, name: str, image_id: str, source: Any = None) -> None:
329
+ self.client.call("prepare", {"tool": name, "image_id": image_id,
330
+ "source": str(source) if source else None})
331
+
332
+ def invoke(self, name: str, payload: dict[str, Any]) -> dict[str, Any]:
333
+ # The route's tool name last, so nothing in the payload can replace
334
+ # it. It used to come first: a request to an allowed tool carrying
335
+ # params.tool="detect.model" ran the model-file loader over HTTP.
336
+ return self.client.call("invoke", {**payload, "tool": name})
337
+
338
+ def close(self) -> None:
339
+ self.client.close()
340
+
341
+
342
+ #: This server's secrets. No child needs them, and the children -- the
343
+ #: sidecar, a training run, the installer -- run third-party code.
344
+ PRIVATE_ENV = frozenset({"EASYANNOTATE_LOCAL_TOKEN", "DATAFACTORY_API_KEY"})
345
+
346
+
347
+ def child_environment(**extra: str) -> dict[str, str]:
348
+ """This process's environment minus its secrets, plus ``extra``."""
349
+ environment = {key: value for key, value in os.environ.items() if key not in PRIVATE_ENV}
350
+ # So CUDA numbers GPUs as nvidia-smi does, and gpu_index means one card.
351
+ environment.setdefault("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
352
+ environment.update(extra)
353
+ return environment
354
+
355
+
356
+ def _salvage(line: bytes) -> dict[str, Any] | None:
357
+ """The last protocol frame inside a line that also carries junk."""
358
+ text = line.decode("utf-8", errors="replace")
359
+ start = text.rfind('{"v"')
360
+ if start <= 0:
361
+ return None
362
+ try:
363
+ return decode_message(text[start:])
364
+ except ToolError:
365
+ return None
366
+
367
+
368
+ def python_for_tools() -> str:
369
+ """Which interpreter runs the sidecar.
370
+
371
+ Deliberately ordered so an explicit choice always wins, and so a tools
372
+ environment kept apart from the main one is found without configuration:
373
+
374
+ 1. ``DATAFACTORY_TOOLS_PYTHON``
375
+ 2. a conventional venv at ``~/.datafactory/tools``
376
+ 3. this interpreter -- correct when someone installed the extra into the
377
+ same environment on purpose
378
+ """
379
+ explicit = os.environ.get("DATAFACTORY_TOOLS_PYTHON")
380
+ if explicit:
381
+ return explicit
382
+ # DATAFACTORY_TOOLS_HOME moves the conventional environment -- and lets
383
+ # the test suite point it somewhere empty, so no test ever finds the real
384
+ # sidecar a developer has installed and spawns PyTorch by accident.
385
+ candidate = Path(os.environ.get("DATAFACTORY_TOOLS_HOME") or Path.home() / ".datafactory" / "tools")
386
+ binary = candidate / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
387
+ if binary.is_file():
388
+ return str(binary)
389
+ return sys.executable