easyannotate 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_factory/__init__.py +82 -0
- data_factory/__main__.py +6 -0
- data_factory/annotate/__init__.py +15 -0
- data_factory/annotate/attach.py +398 -0
- data_factory/annotate/demo.py +161 -0
- data_factory/annotate/images.py +443 -0
- data_factory/annotate/labels.py +399 -0
- data_factory/annotate/loop.py +359 -0
- data_factory/annotate/projects.py +207 -0
- data_factory/annotate/review.py +471 -0
- data_factory/annotate/schema.sql +171 -0
- data_factory/annotate/shapes.py +488 -0
- data_factory/annotate/store.py +292 -0
- data_factory/annotate/tools/__init__.py +39 -0
- data_factory/annotate/tools/fake.py +170 -0
- data_factory/annotate/tools/install.py +254 -0
- data_factory/annotate/tools/local.py +389 -0
- data_factory/annotate/tools/postprocess.py +175 -0
- data_factory/annotate/tools/protocol.py +238 -0
- data_factory/annotate/tools/registry.py +196 -0
- data_factory/annotate/tools/results.py +83 -0
- data_factory/annotate/tools/serve.py +106 -0
- data_factory/cli/__init__.py +21 -0
- data_factory/cli/main.py +837 -0
- data_factory/cli/menu.py +173 -0
- data_factory/cli/render.py +158 -0
- data_factory/core/__init__.py +11 -0
- data_factory/core/annotations/__init__.py +5 -0
- data_factory/core/annotations/coco.py +394 -0
- data_factory/core/annotations/csvfmt.py +174 -0
- data_factory/core/annotations/dota.py +98 -0
- data_factory/core/annotations/formats.py +626 -0
- data_factory/core/annotations/labelme.py +153 -0
- data_factory/core/annotations/model.py +334 -0
- data_factory/core/annotations/voc.py +231 -0
- data_factory/core/annotations/yolo.py +283 -0
- data_factory/core/bktree.py +89 -0
- data_factory/core/fs.py +245 -0
- data_factory/core/imaging.py +626 -0
- data_factory/core/ops/__init__.py +44 -0
- data_factory/core/ops/analyze.py +383 -0
- data_factory/core/ops/annotate_export.py +326 -0
- data_factory/core/ops/audit.py +274 -0
- data_factory/core/ops/augment.py +454 -0
- data_factory/core/ops/collect.py +241 -0
- data_factory/core/ops/combine.py +582 -0
- data_factory/core/ops/convert.py +212 -0
- data_factory/core/ops/dataset.py +246 -0
- data_factory/core/ops/dedup.py +235 -0
- data_factory/core/ops/export.py +168 -0
- data_factory/core/ops/image.py +407 -0
- data_factory/core/ops/labels.py +215 -0
- data_factory/core/ops/pairing.py +133 -0
- data_factory/core/ops/rename.py +154 -0
- data_factory/core/ops/report.py +323 -0
- data_factory/core/ops/visualize.py +164 -0
- data_factory/core/pairs.py +144 -0
- data_factory/core/parallel.py +191 -0
- data_factory/core/provenance.py +136 -0
- data_factory/core/registry.py +339 -0
- data_factory/core/stats.py +403 -0
- data_factory/core/types.py +148 -0
- data_factory/core/writer.py +194 -0
- data_factory/desktop.py +207 -0
- data_factory/hardware.py +243 -0
- data_factory/pipeline.py +192 -0
- data_factory/py.typed +0 -0
- data_factory/service/__init__.py +26 -0
- data_factory/service/app.py +483 -0
- data_factory/service/binding.py +174 -0
- data_factory/service/deps.py +133 -0
- data_factory/service/jobs.py +177 -0
- data_factory/service/limits.py +71 -0
- data_factory/service/routers/__init__.py +11 -0
- data_factory/service/routers/annotate.py +582 -0
- data_factory/service/routers/local.py +464 -0
- data_factory/service/routers/media.py +154 -0
- data_factory/service/routers/predictions.py +116 -0
- data_factory/service/routers/promote.py +182 -0
- data_factory/service/routers/tools.py +308 -0
- data_factory/service/static/annotate/api.js +246 -0
- data_factory/service/static/annotate/app.js +1696 -0
- data_factory/service/static/annotate/assist.js +106 -0
- data_factory/service/static/annotate/filmstrip.js +286 -0
- data_factory/service/static/annotate/hit.js +146 -0
- data_factory/service/static/annotate/labels.js +197 -0
- data_factory/service/static/annotate/render.js +286 -0
- data_factory/service/static/annotate/scene.js +277 -0
- data_factory/service/static/annotate/shapelist.js +72 -0
- data_factory/service/static/annotate/tools/box.js +96 -0
- data_factory/service/static/annotate/tools/line.js +96 -0
- data_factory/service/static/annotate/tools/obb.js +88 -0
- data_factory/service/static/annotate/tools/point.js +49 -0
- data_factory/service/static/annotate/tools/polygon.js +110 -0
- data_factory/service/static/annotate/tools/select.js +227 -0
- data_factory/service/static/annotate/undo.js +176 -0
- data_factory/service/static/annotate/viewport.js +156 -0
- data_factory/service/static/annotate.html +390 -0
- data_factory/service/static/index.html +632 -0
- data_factory/service/workspace.py +244 -0
- easyannotate/__init__.py +64 -0
- easyannotate/__main__.py +5 -0
- easyannotate/project.py +181 -0
- easyannotate/py.typed +0 -0
- easyannotate-0.4.0.dist-info/METADATA +767 -0
- easyannotate-0.4.0.dist-info/RECORD +109 -0
- easyannotate-0.4.0.dist-info/WHEEL +4 -0
- easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
- easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
"""Installing the model tools: one environment, chosen for this machine.
|
|
2
|
+
|
|
3
|
+
Shared by ``easyannotate tools install`` and the desktop/local web app's
|
|
4
|
+
"Install AI tools" button. Nothing heavy is downloaded until someone asks.
|
|
5
|
+
|
|
6
|
+
The environment lives at ``~/.datafactory/tools`` (``DATAFACTORY_TOOLS_HOME``),
|
|
7
|
+
apart from the editor's: torch's version matrix is resolved there alone, and
|
|
8
|
+
a broken install there breaks nothing but the AI buttons.
|
|
9
|
+
|
|
10
|
+
Where the pieces come from, in order of preference:
|
|
11
|
+
|
|
12
|
+
* **uv** -- bundled inside the desktop app, else on PATH. It creates the
|
|
13
|
+
environment and fetches a Python for it, so a frozen app with no Python of
|
|
14
|
+
its own can still install. Without uv, a pip install falls back to venv+pip.
|
|
15
|
+
* **easyannotate + easyannotate-tools** -- wheels bundled with the desktop app
|
|
16
|
+
(``EASYANNOTATE_WHEELS`` or the app's ``wheels/`` folder), else a source
|
|
17
|
+
checkout (``--source``), else PyPI.
|
|
18
|
+
* **PyTorch** -- the build ``hardware.plan()`` picked: the CUDA version this
|
|
19
|
+
machine's *driver* supports, Metal on Apple silicon, or CPU.
|
|
20
|
+
|
|
21
|
+
A ``profile.json`` beside the environment records the device and model sizes,
|
|
22
|
+
and the sidecar reads it, so the same code runs SAM tiny on a laptop CPU and
|
|
23
|
+
SAM base on a 12 GB card.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import json
|
|
29
|
+
import os
|
|
30
|
+
import shutil
|
|
31
|
+
import subprocess
|
|
32
|
+
import sys
|
|
33
|
+
from collections.abc import Callable
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
from typing import Any
|
|
36
|
+
|
|
37
|
+
from ... import hardware
|
|
38
|
+
from .local import child_environment, python_for_tools
|
|
39
|
+
|
|
40
|
+
PROFILE = "profile.json"
|
|
41
|
+
PYTHON_VERSION = "3.12"
|
|
42
|
+
|
|
43
|
+
#: Run in the new environment. Real work on the device, not
|
|
44
|
+
#: torch.cuda.is_available(): measured on a 529 driver with the cu126 build,
|
|
45
|
+
#: that said True while every allocation failed ("device busy or unavailable").
|
|
46
|
+
_PROBE = '''
|
|
47
|
+
import json, sys
|
|
48
|
+
out = {"device": "cpu", "error": ""}
|
|
49
|
+
import torch
|
|
50
|
+
out["torch"] = torch.__version__
|
|
51
|
+
def works(device):
|
|
52
|
+
x = torch.randn(128, 128, device=device)
|
|
53
|
+
float((x @ x).sum())
|
|
54
|
+
try:
|
|
55
|
+
if torch.cuda.is_available():
|
|
56
|
+
works("cuda:%d" % int(sys.argv[1]))
|
|
57
|
+
out["device"] = "cuda"
|
|
58
|
+
elif getattr(torch.backends, "mps", None) and torch.backends.mps.is_available():
|
|
59
|
+
works("mps")
|
|
60
|
+
out["device"] = "mps"
|
|
61
|
+
except Exception as exc:
|
|
62
|
+
out["error"] = (type(exc).__name__ + ": " + str(exc)).splitlines()[0][:300]
|
|
63
|
+
print("PROBE " + json.dumps(out))
|
|
64
|
+
'''
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class InstallError(RuntimeError):
|
|
68
|
+
pass
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def tools_home() -> Path:
|
|
72
|
+
return Path(os.environ.get("DATAFACTORY_TOOLS_HOME") or Path.home() / ".datafactory" / "tools")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def tools_python(home: Path | None = None) -> Path:
|
|
76
|
+
home = home or tools_home()
|
|
77
|
+
return home / ("Scripts/python.exe" if sys.platform == "win32" else "bin/python")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def read_profile(home: Path | None = None) -> dict[str, Any]:
|
|
81
|
+
path = (home or tools_home()) / PROFILE
|
|
82
|
+
try:
|
|
83
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
84
|
+
except (OSError, ValueError):
|
|
85
|
+
return {}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def status() -> dict[str, Any]:
|
|
89
|
+
"""Installed or not, and what this machine would get."""
|
|
90
|
+
hw = hardware.detect()
|
|
91
|
+
chosen = hardware.plan(hw)
|
|
92
|
+
profile = read_profile()
|
|
93
|
+
return {
|
|
94
|
+
"installed": tools_python().is_file() and bool(profile),
|
|
95
|
+
"profile": profile,
|
|
96
|
+
"hardware": hw.as_dict(),
|
|
97
|
+
"plan": chosen.as_dict(),
|
|
98
|
+
"uv": bool(find_uv()),
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def find_uv() -> str | None:
|
|
103
|
+
bundled = Path(getattr(sys, "_MEIPASS", "")) / "bin" / ("uv.exe" if sys.platform == "win32" else "uv")
|
|
104
|
+
if getattr(sys, "_MEIPASS", None) and bundled.is_file():
|
|
105
|
+
return str(bundled)
|
|
106
|
+
return shutil.which("uv")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def bundled_wheels() -> list[Path]:
|
|
110
|
+
folders = [os.environ.get("EASYANNOTATE_WHEELS", "")]
|
|
111
|
+
if getattr(sys, "_MEIPASS", None):
|
|
112
|
+
folders.append(str(Path(sys._MEIPASS) / "wheels")) # noqa: SLF001
|
|
113
|
+
for folder in folders:
|
|
114
|
+
if folder and Path(folder).is_dir():
|
|
115
|
+
found = sorted(Path(folder).glob("*.whl"))
|
|
116
|
+
if found:
|
|
117
|
+
return found
|
|
118
|
+
return []
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def install(
|
|
122
|
+
*,
|
|
123
|
+
source: Path | None = None,
|
|
124
|
+
device: str | None = None,
|
|
125
|
+
on_line: Callable[[str], None] = print,
|
|
126
|
+
) -> dict[str, Any]:
|
|
127
|
+
"""Create or update the tools environment. Returns the written profile.
|
|
128
|
+
|
|
129
|
+
``device`` forces "cuda", "mps" or "cpu" instead of the detected choice.
|
|
130
|
+
"""
|
|
131
|
+
hw = hardware.detect()
|
|
132
|
+
chosen = hardware.plan(hw)
|
|
133
|
+
if device == "cpu" and chosen.device != "cpu":
|
|
134
|
+
chosen = hardware.plan(hardware.Hardware(os=hw.os, cpu_count=hw.cpu_count, ram_gb=hw.ram_gb))
|
|
135
|
+
elif device in ("cuda", "mps") and chosen.device != device:
|
|
136
|
+
raise InstallError(f"asked for {device}, but this machine looks like: {chosen.summary}")
|
|
137
|
+
for warning in chosen.warnings:
|
|
138
|
+
on_line(f"note: {warning}")
|
|
139
|
+
on_line(f"plan: {chosen.summary} (about {chosen.download_gb:g} GB)")
|
|
140
|
+
|
|
141
|
+
home = tools_home()
|
|
142
|
+
python = tools_python(home)
|
|
143
|
+
uv = find_uv()
|
|
144
|
+
# Gone until this install has verified a working PyTorch: a half-finished
|
|
145
|
+
# install must not still read as "installed" with the old device.
|
|
146
|
+
(home / PROFILE).unlink(missing_ok=True)
|
|
147
|
+
|
|
148
|
+
def run(command: list[str]) -> None:
|
|
149
|
+
on_line("$ " + " ".join(str(part) for part in command))
|
|
150
|
+
try:
|
|
151
|
+
process = subprocess.Popen(
|
|
152
|
+
[str(part) for part in command],
|
|
153
|
+
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
|
|
154
|
+
encoding="utf-8", errors="replace", env=child_environment(),
|
|
155
|
+
)
|
|
156
|
+
except OSError as exc:
|
|
157
|
+
raise InstallError(f"could not start {command[0]}: {exc}") from None
|
|
158
|
+
assert process.stdout is not None
|
|
159
|
+
try:
|
|
160
|
+
for line in process.stdout:
|
|
161
|
+
line = line.rstrip()
|
|
162
|
+
if line:
|
|
163
|
+
on_line(line)
|
|
164
|
+
code = process.wait()
|
|
165
|
+
finally:
|
|
166
|
+
# Cancelled, or the server stopping: never leave pip running on.
|
|
167
|
+
if process.poll() is None:
|
|
168
|
+
process.kill()
|
|
169
|
+
process.wait()
|
|
170
|
+
if code != 0:
|
|
171
|
+
raise InstallError(f"failed: {' '.join(str(part) for part in command[:4])} ...")
|
|
172
|
+
|
|
173
|
+
if not python.is_file():
|
|
174
|
+
on_line(f"creating {home}")
|
|
175
|
+
if uv:
|
|
176
|
+
run([uv, "venv", str(home), "--python", PYTHON_VERSION])
|
|
177
|
+
elif not getattr(sys, "frozen", False):
|
|
178
|
+
run([sys.executable, "-m", "venv", str(home)])
|
|
179
|
+
else:
|
|
180
|
+
raise InstallError("this build has no uv to create the environment with")
|
|
181
|
+
pip = [uv, "pip", "install", "--python", str(python)] if uv else [str(python), "-m", "pip", "install"]
|
|
182
|
+
|
|
183
|
+
if chosen.torch_index:
|
|
184
|
+
on_line("installing PyTorch -- the large part of the download")
|
|
185
|
+
# Reinstall, not install: a PyTorch already there -- the CPU build,
|
|
186
|
+
# say -- "satisfies" plain `torch`, so asking for the CUDA build would
|
|
187
|
+
# quietly change nothing. Measured: that is exactly what happened.
|
|
188
|
+
force = (
|
|
189
|
+
["--reinstall-package", "torch", "--reinstall-package", "torchvision"]
|
|
190
|
+
if uv else ["--force-reinstall"]
|
|
191
|
+
)
|
|
192
|
+
packages = list(chosen.torch_packages) or ["torch", "torchvision"]
|
|
193
|
+
run(pip + [*packages, "--index-url", chosen.torch_index, *force])
|
|
194
|
+
|
|
195
|
+
wheels = bundled_wheels()
|
|
196
|
+
if wheels:
|
|
197
|
+
packages = [str(w) for w in wheels]
|
|
198
|
+
elif source is not None:
|
|
199
|
+
packages = ["-e", str(source), "-e", str(Path(source) / "tools")]
|
|
200
|
+
else:
|
|
201
|
+
# Pinned to this version: the two are released together, and an
|
|
202
|
+
# unpinned name is whatever PyPI says it is today.
|
|
203
|
+
from ... import __version__
|
|
204
|
+
|
|
205
|
+
packages = [f"easyannotate-tools=={__version__}"]
|
|
206
|
+
on_line("installing the EasyAnnotate tools and Ultralytics")
|
|
207
|
+
run(pip + packages)
|
|
208
|
+
|
|
209
|
+
profile = {
|
|
210
|
+
"device": chosen.device,
|
|
211
|
+
"sam": chosen.sam,
|
|
212
|
+
"world": chosen.world,
|
|
213
|
+
"summary": chosen.summary,
|
|
214
|
+
"gpu_index": chosen.gpu_index,
|
|
215
|
+
"hardware": hw.as_dict(),
|
|
216
|
+
}
|
|
217
|
+
on_line("checking that PyTorch runs on this machine")
|
|
218
|
+
try:
|
|
219
|
+
probe = subprocess.run(
|
|
220
|
+
[str(python), "-I", "-c", _PROBE, str(chosen.gpu_index)],
|
|
221
|
+
capture_output=True, text=True, encoding="utf-8", errors="replace",
|
|
222
|
+
timeout=600, env=child_environment(),
|
|
223
|
+
)
|
|
224
|
+
except subprocess.TimeoutExpired:
|
|
225
|
+
raise InstallError("PyTorch took over 10 minutes to start -- try again, or `tools install --cpu`") from None
|
|
226
|
+
# rfind, not startswith: a library's output without a newline can come
|
|
227
|
+
# first on the same line (see serve._private_stdout).
|
|
228
|
+
output = probe.stdout or ""
|
|
229
|
+
start = output.rfind("PROBE {")
|
|
230
|
+
if probe.returncode != 0 or start < 0:
|
|
231
|
+
tail = " ".join((probe.stderr or "").strip().splitlines()[-2:])
|
|
232
|
+
raise InstallError(f"PyTorch does not start in {home}: {tail or 'no output'}")
|
|
233
|
+
try:
|
|
234
|
+
result = json.loads(output[start + len("PROBE "):].splitlines()[0])
|
|
235
|
+
except ValueError:
|
|
236
|
+
raise InstallError(f"PyTorch's self-check in {home} printed something unreadable") from None
|
|
237
|
+
seen = result.get("device") or "cpu"
|
|
238
|
+
profile["verified_device"] = seen
|
|
239
|
+
profile["torch"] = result.get("torch", "")
|
|
240
|
+
if chosen.device != "cpu" and seen != chosen.device:
|
|
241
|
+
why = result.get("error") or f"PyTorch sees {seen}"
|
|
242
|
+
on_line(f"warning: the {chosen.device} build does not run here ({why}) -- the tools will use the CPU")
|
|
243
|
+
# Then use the sizes a CPU can carry, and say what really happened.
|
|
244
|
+
cpu = hardware.plan(hardware.Hardware(os=hw.os, cpu_count=hw.cpu_count, ram_gb=hw.ram_gb))
|
|
245
|
+
profile.update({
|
|
246
|
+
"device": "cpu", "sam": cpu.sam, "world": cpu.world,
|
|
247
|
+
"summary": f"{cpu.summary} (the GPU build did not load -- see the log)",
|
|
248
|
+
})
|
|
249
|
+
(home / PROFILE).write_text(json.dumps(profile, indent=2), encoding="utf-8")
|
|
250
|
+
on_line(f"done: {profile['summary']} (PyTorch sees: {seen})")
|
|
251
|
+
return profile
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
__all__ = ["InstallError", "install", "python_for_tools", "read_profile", "status", "tools_home"]
|
|
@@ -0,0 +1,389 @@
|
|
|
1
|
+
"""A heavy tool running as a child process, spoken to over stdio.
|
|
2
|
+
|
|
3
|
+
The legacy SAM script found this boundary by accident -- a ``multiprocessing``
|
|
4
|
+
worker holding two predictors -- and it was the best idea in either annotation
|
|
5
|
+
script. This formalises it, and fixes the three things that made the original
|
|
6
|
+
fragile:
|
|
7
|
+
|
|
8
|
+
* **A blocking read inside the UI callback.** Here a background thread owns the
|
|
9
|
+
pipe and dispatches replies to futures, so no handler ever blocks on a raw
|
|
10
|
+
read, and every wait has a timeout.
|
|
11
|
+
* **An untagged queue.** The original had one output queue and no correlation,
|
|
12
|
+
so a stale reply could be consumed by the wrong caller and a real result
|
|
13
|
+
silently dropped. Every message here carries an id and is matched on it.
|
|
14
|
+
* **An exception path that never replied.** The original caught, printed, and
|
|
15
|
+
put nothing back, so the UI waited forever. Here the child always answers and
|
|
16
|
+
the parent additionally fails every outstanding call if the child dies.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import os
|
|
22
|
+
import subprocess
|
|
23
|
+
import sys
|
|
24
|
+
import threading
|
|
25
|
+
import time
|
|
26
|
+
from concurrent.futures import Future
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
from typing import Any
|
|
29
|
+
|
|
30
|
+
from .protocol import (
|
|
31
|
+
DEFAULT_TIMEOUTS,
|
|
32
|
+
ToolError,
|
|
33
|
+
ToolHealth,
|
|
34
|
+
ToolSpec,
|
|
35
|
+
ToolUnavailable,
|
|
36
|
+
decode_message,
|
|
37
|
+
encode_message,
|
|
38
|
+
raise_for,
|
|
39
|
+
request,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
#: Restarts before we stop trying. A tool that dies three times is broken, and
|
|
43
|
+
#: retrying forever would hide that while burning CPU.
|
|
44
|
+
MAX_RESTARTS = 3
|
|
45
|
+
RESTART_BACKOFF = (0.5, 2.0, 5.0)
|
|
46
|
+
|
|
47
|
+
#: Keep the tail of the child's stderr so a failure can say *why* rather than
|
|
48
|
+
#: just "it died" -- the difference between a fixable report and a shrug.
|
|
49
|
+
STDERR_LINES = 40
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class SidecarClient:
|
|
53
|
+
"""Owns one child process and turns its stdio into method calls."""
|
|
54
|
+
|
|
55
|
+
def __init__(
|
|
56
|
+
self,
|
|
57
|
+
argv: list[str],
|
|
58
|
+
*,
|
|
59
|
+
cwd: Path | None = None,
|
|
60
|
+
env: dict[str, str] | None = None,
|
|
61
|
+
timeouts: dict[str, float] | None = None,
|
|
62
|
+
) -> None:
|
|
63
|
+
self.argv = list(argv)
|
|
64
|
+
self.cwd = cwd
|
|
65
|
+
self.env = env
|
|
66
|
+
self.timeouts = {**DEFAULT_TIMEOUTS, **(timeouts or {})}
|
|
67
|
+
|
|
68
|
+
self._lock = threading.Lock()
|
|
69
|
+
self._process: subprocess.Popen[bytes] | None = None
|
|
70
|
+
self._reader: threading.Thread | None = None
|
|
71
|
+
self._stderr_reader: threading.Thread | None = None
|
|
72
|
+
self._pending: dict[int, Future] = {}
|
|
73
|
+
self._stderr: list[str] = []
|
|
74
|
+
self._next_id = 0
|
|
75
|
+
self._restarts = 0
|
|
76
|
+
self._closed = False
|
|
77
|
+
self._last_error = ""
|
|
78
|
+
|
|
79
|
+
# -- lifecycle ---------------------------------------------------------
|
|
80
|
+
def start(self) -> None:
|
|
81
|
+
"""Spawn the child. Raises ToolUnavailable if it will not start."""
|
|
82
|
+
with self._lock:
|
|
83
|
+
if self._closed:
|
|
84
|
+
raise ToolUnavailable("this client has been closed")
|
|
85
|
+
if self._process is not None and self._process.poll() is None:
|
|
86
|
+
return
|
|
87
|
+
self._spawn_locked()
|
|
88
|
+
|
|
89
|
+
def _spawn_locked(self) -> None:
|
|
90
|
+
environment = {**child_environment(), **(self.env or {})}
|
|
91
|
+
# Unbuffered, or the child's replies sit in a pipe buffer and every
|
|
92
|
+
# call looks like a timeout.
|
|
93
|
+
environment["PYTHONUNBUFFERED"] = "1"
|
|
94
|
+
try:
|
|
95
|
+
self._process = subprocess.Popen(
|
|
96
|
+
self.argv,
|
|
97
|
+
stdin=subprocess.PIPE,
|
|
98
|
+
stdout=subprocess.PIPE,
|
|
99
|
+
stderr=subprocess.PIPE,
|
|
100
|
+
cwd=str(self.cwd) if self.cwd else None,
|
|
101
|
+
env=environment,
|
|
102
|
+
bufsize=0,
|
|
103
|
+
)
|
|
104
|
+
except (OSError, ValueError) as exc:
|
|
105
|
+
self._last_error = str(exc)
|
|
106
|
+
raise ToolUnavailable(f"cannot start {self.argv[0]!r}: {exc}") from None
|
|
107
|
+
|
|
108
|
+
self._stderr = []
|
|
109
|
+
self._reader = threading.Thread(target=self._read_stdout, args=(self._process,),
|
|
110
|
+
daemon=True, name="dfx-tool-out")
|
|
111
|
+
self._stderr_reader = threading.Thread(target=self._read_stderr, args=(self._process,),
|
|
112
|
+
daemon=True, name="dfx-tool-err")
|
|
113
|
+
self._reader.start()
|
|
114
|
+
self._stderr_reader.start()
|
|
115
|
+
|
|
116
|
+
def close(self) -> None:
|
|
117
|
+
with self._lock:
|
|
118
|
+
self._closed = True
|
|
119
|
+
process = self._process
|
|
120
|
+
self._process = None
|
|
121
|
+
if process is None:
|
|
122
|
+
return
|
|
123
|
+
try:
|
|
124
|
+
if process.stdin:
|
|
125
|
+
process.stdin.close()
|
|
126
|
+
process.wait(timeout=2)
|
|
127
|
+
except Exception:
|
|
128
|
+
process.kill()
|
|
129
|
+
finally:
|
|
130
|
+
self._fail_pending(ToolUnavailable("the tool was shut down"))
|
|
131
|
+
|
|
132
|
+
@property
|
|
133
|
+
def alive(self) -> bool:
|
|
134
|
+
process = self._process
|
|
135
|
+
return process is not None and process.poll() is None
|
|
136
|
+
|
|
137
|
+
# -- calling -----------------------------------------------------------
|
|
138
|
+
def call(self, method: str, params: dict[str, Any] | None = None,
|
|
139
|
+
*, timeout: float | None = None) -> Any:
|
|
140
|
+
"""Send one request and wait for its reply.
|
|
141
|
+
|
|
142
|
+
Restarts the child once per call if it has died, so a crashed tool
|
|
143
|
+
recovers on the next click rather than staying broken until a restart.
|
|
144
|
+
"""
|
|
145
|
+
if self._closed:
|
|
146
|
+
raise ToolUnavailable("this client has been closed")
|
|
147
|
+
wait = timeout if timeout is not None else self.timeouts.get(method, 15.0)
|
|
148
|
+
|
|
149
|
+
# Two attempts, because the child can die *between* the liveness check
|
|
150
|
+
# and the write -- a crash on the previous call is reaped
|
|
151
|
+
# asynchronously, so `poll()` can still say None for a moment while the
|
|
152
|
+
# pipe is already gone. Without the retry the caller sees a confusing
|
|
153
|
+
# "[Errno 22] Invalid argument" for a tool that is perfectly able to
|
|
154
|
+
# restart and serve the request.
|
|
155
|
+
last: Exception | None = None
|
|
156
|
+
for _ in range(2):
|
|
157
|
+
if not self.alive:
|
|
158
|
+
self._restart() # raises once restarts are exhausted
|
|
159
|
+
|
|
160
|
+
message_id, future, payload = self._enqueue(method, params)
|
|
161
|
+
try:
|
|
162
|
+
self._write(payload)
|
|
163
|
+
except ToolUnavailable as exc:
|
|
164
|
+
self._drop(message_id)
|
|
165
|
+
last = exc
|
|
166
|
+
self._kill_child() # force a fresh child on the next pass
|
|
167
|
+
continue
|
|
168
|
+
|
|
169
|
+
try:
|
|
170
|
+
reply = future.result(timeout=wait)
|
|
171
|
+
except TimeoutError:
|
|
172
|
+
self._drop(message_id)
|
|
173
|
+
# A tool that misses its deadline is not trustworthy
|
|
174
|
+
# afterwards: whatever it is still doing is unknown, and a late
|
|
175
|
+
# reply would arrive against an id nobody is waiting for.
|
|
176
|
+
# Restarting is the honest recovery -- except for health,
|
|
177
|
+
# which is only a question: a busy sidecar is not a broken one,
|
|
178
|
+
# and killing it on a status probe threw away loaded models.
|
|
179
|
+
if method != "health":
|
|
180
|
+
self._kill_child()
|
|
181
|
+
# The sidecar's own last words are the only clue there is.
|
|
182
|
+
tail = self._stderr_tail()
|
|
183
|
+
raise ToolError(
|
|
184
|
+
f"{method} took longer than {wait:g}s" + (f" -- sidecar said: {tail}" if tail else ""),
|
|
185
|
+
code="timeout",
|
|
186
|
+
) from None
|
|
187
|
+
# A child that answered is working: its restart budget is earned
|
|
188
|
+
# back, so three crashes over a day do not disable the tools.
|
|
189
|
+
self._restarts = 0
|
|
190
|
+
return raise_for(reply)
|
|
191
|
+
|
|
192
|
+
raise last or ToolUnavailable("the tool could not be reached")
|
|
193
|
+
|
|
194
|
+
def _enqueue(self, method: str, params: dict[str, Any] | None):
|
|
195
|
+
with self._lock:
|
|
196
|
+
self._next_id += 1
|
|
197
|
+
message_id = self._next_id
|
|
198
|
+
future: Future = Future()
|
|
199
|
+
self._pending[message_id] = future
|
|
200
|
+
return message_id, future, request(message_id, method, params)
|
|
201
|
+
|
|
202
|
+
def _drop(self, message_id: int) -> None:
|
|
203
|
+
with self._lock:
|
|
204
|
+
self._pending.pop(message_id, None)
|
|
205
|
+
|
|
206
|
+
def _write(self, payload: dict[str, Any]) -> None:
|
|
207
|
+
process = self._process
|
|
208
|
+
if process is None or process.stdin is None or process.poll() is not None:
|
|
209
|
+
raise ToolUnavailable("the tool is not running")
|
|
210
|
+
try:
|
|
211
|
+
process.stdin.write(encode_message(payload))
|
|
212
|
+
process.stdin.flush()
|
|
213
|
+
except (BrokenPipeError, OSError) as exc:
|
|
214
|
+
raise ToolUnavailable(f"lost the tool while writing: {exc}") from None
|
|
215
|
+
|
|
216
|
+
# -- reading -----------------------------------------------------------
|
|
217
|
+
def _read_stdout(self, process: subprocess.Popen[bytes]) -> None:
|
|
218
|
+
stream = process.stdout
|
|
219
|
+
assert stream is not None
|
|
220
|
+
try:
|
|
221
|
+
for line in stream:
|
|
222
|
+
if not line.strip():
|
|
223
|
+
continue
|
|
224
|
+
try:
|
|
225
|
+
payload = decode_message(line)
|
|
226
|
+
except ToolError:
|
|
227
|
+
# Junk on stdout is a tool that printed instead of
|
|
228
|
+
# replying -- often a progress bar with no newline, with
|
|
229
|
+
# the real reply glued to its end. Salvage that; skip
|
|
230
|
+
# the rest rather than killing the session.
|
|
231
|
+
payload = _salvage(line)
|
|
232
|
+
if payload is None:
|
|
233
|
+
continue
|
|
234
|
+
try:
|
|
235
|
+
self._resolve(payload)
|
|
236
|
+
except Exception: # noqa: BLE001 -- one bad frame must not stop the reader
|
|
237
|
+
continue
|
|
238
|
+
except (ValueError, OSError):
|
|
239
|
+
pass
|
|
240
|
+
finally:
|
|
241
|
+
# The child is gone. Nothing else will ever answer the calls still
|
|
242
|
+
# waiting, so fail them now instead of letting each hit its
|
|
243
|
+
# timeout one after another.
|
|
244
|
+
self._fail_pending(
|
|
245
|
+
ToolUnavailable(f"the tool exited{self._stderr_tail()}")
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
def _read_stderr(self, process: subprocess.Popen[bytes]) -> None:
|
|
249
|
+
stream = process.stderr
|
|
250
|
+
assert stream is not None
|
|
251
|
+
try:
|
|
252
|
+
for line in stream:
|
|
253
|
+
text = line.decode("utf-8", errors="replace").rstrip()
|
|
254
|
+
if not text:
|
|
255
|
+
continue
|
|
256
|
+
self._stderr.append(text)
|
|
257
|
+
del self._stderr[:-STDERR_LINES]
|
|
258
|
+
except (ValueError, OSError):
|
|
259
|
+
pass
|
|
260
|
+
|
|
261
|
+
def _resolve(self, payload: dict[str, Any]) -> None:
|
|
262
|
+
message_id = payload.get("id")
|
|
263
|
+
if not isinstance(message_id, int):
|
|
264
|
+
return
|
|
265
|
+
with self._lock:
|
|
266
|
+
future = self._pending.pop(message_id, None)
|
|
267
|
+
# An unknown id is a late reply to something already abandoned. Drop
|
|
268
|
+
# it: matching by id is exactly what stops it poisoning the next call.
|
|
269
|
+
if future is not None and not future.done():
|
|
270
|
+
future.set_result(payload)
|
|
271
|
+
|
|
272
|
+
def _fail_pending(self, error: Exception) -> None:
|
|
273
|
+
with self._lock:
|
|
274
|
+
pending, self._pending = self._pending, {}
|
|
275
|
+
for future in pending.values():
|
|
276
|
+
if not future.done():
|
|
277
|
+
future.set_exception(error)
|
|
278
|
+
|
|
279
|
+
def _stderr_tail(self) -> str:
|
|
280
|
+
if not self._stderr:
|
|
281
|
+
return ""
|
|
282
|
+
return ": " + " / ".join(self._stderr[-3:])
|
|
283
|
+
|
|
284
|
+
def _kill_child(self) -> None:
|
|
285
|
+
with self._lock:
|
|
286
|
+
process, self._process = self._process, None
|
|
287
|
+
if process is not None:
|
|
288
|
+
try:
|
|
289
|
+
process.kill()
|
|
290
|
+
except Exception:
|
|
291
|
+
pass
|
|
292
|
+
|
|
293
|
+
def _restart(self) -> None:
|
|
294
|
+
if self._restarts >= MAX_RESTARTS:
|
|
295
|
+
raise ToolUnavailable(
|
|
296
|
+
f"the tool has died {self._restarts} times and will not be restarted"
|
|
297
|
+
f"{self._stderr_tail()}"
|
|
298
|
+
)
|
|
299
|
+
delay = RESTART_BACKOFF[min(self._restarts, len(RESTART_BACKOFF) - 1)]
|
|
300
|
+
self._restarts += 1
|
|
301
|
+
if self._restarts > 1:
|
|
302
|
+
time.sleep(delay)
|
|
303
|
+
with self._lock:
|
|
304
|
+
self._spawn_locked()
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
class LocalTool:
|
|
308
|
+
"""A :class:`Tool` backed by a sidecar process."""
|
|
309
|
+
|
|
310
|
+
def __init__(self, argv: list[str], **kwargs: Any) -> None:
|
|
311
|
+
self.client = SidecarClient(argv, **kwargs)
|
|
312
|
+
self._specs: list[ToolSpec] | None = None
|
|
313
|
+
|
|
314
|
+
def describe(self) -> list[ToolSpec]:
|
|
315
|
+
if self._specs is None:
|
|
316
|
+
raw = self.client.call("describe")
|
|
317
|
+
self._specs = [ToolSpec.from_dict(item) for item in (raw or [])]
|
|
318
|
+
return list(self._specs)
|
|
319
|
+
|
|
320
|
+
def health(self, name: str = "") -> ToolHealth:
|
|
321
|
+
try:
|
|
322
|
+
return ToolHealth.from_dict(self.client.call("health", {"tool": name}) or {})
|
|
323
|
+
except ToolError as exc:
|
|
324
|
+
# Health must never raise: it is what the UI calls to decide
|
|
325
|
+
# whether to show a button at all.
|
|
326
|
+
return ToolHealth("unavailable", str(exc))
|
|
327
|
+
|
|
328
|
+
def prepare(self, name: str, image_id: str, source: Any = None) -> None:
|
|
329
|
+
self.client.call("prepare", {"tool": name, "image_id": image_id,
|
|
330
|
+
"source": str(source) if source else None})
|
|
331
|
+
|
|
332
|
+
def invoke(self, name: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|
333
|
+
# The route's tool name last, so nothing in the payload can replace
|
|
334
|
+
# it. It used to come first: a request to an allowed tool carrying
|
|
335
|
+
# params.tool="detect.model" ran the model-file loader over HTTP.
|
|
336
|
+
return self.client.call("invoke", {**payload, "tool": name})
|
|
337
|
+
|
|
338
|
+
def close(self) -> None:
|
|
339
|
+
self.client.close()
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
#: This server's secrets. No child needs them, and the children -- the
|
|
343
|
+
#: sidecar, a training run, the installer -- run third-party code.
|
|
344
|
+
PRIVATE_ENV = frozenset({"EASYANNOTATE_LOCAL_TOKEN", "DATAFACTORY_API_KEY"})
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def child_environment(**extra: str) -> dict[str, str]:
|
|
348
|
+
"""This process's environment minus its secrets, plus ``extra``."""
|
|
349
|
+
environment = {key: value for key, value in os.environ.items() if key not in PRIVATE_ENV}
|
|
350
|
+
# So CUDA numbers GPUs as nvidia-smi does, and gpu_index means one card.
|
|
351
|
+
environment.setdefault("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
|
|
352
|
+
environment.update(extra)
|
|
353
|
+
return environment
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _salvage(line: bytes) -> dict[str, Any] | None:
|
|
357
|
+
"""The last protocol frame inside a line that also carries junk."""
|
|
358
|
+
text = line.decode("utf-8", errors="replace")
|
|
359
|
+
start = text.rfind('{"v"')
|
|
360
|
+
if start <= 0:
|
|
361
|
+
return None
|
|
362
|
+
try:
|
|
363
|
+
return decode_message(text[start:])
|
|
364
|
+
except ToolError:
|
|
365
|
+
return None
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def python_for_tools() -> str:
|
|
369
|
+
"""Which interpreter runs the sidecar.
|
|
370
|
+
|
|
371
|
+
Deliberately ordered so an explicit choice always wins, and so a tools
|
|
372
|
+
environment kept apart from the main one is found without configuration:
|
|
373
|
+
|
|
374
|
+
1. ``DATAFACTORY_TOOLS_PYTHON``
|
|
375
|
+
2. a conventional venv at ``~/.datafactory/tools``
|
|
376
|
+
3. this interpreter -- correct when someone installed the extra into the
|
|
377
|
+
same environment on purpose
|
|
378
|
+
"""
|
|
379
|
+
explicit = os.environ.get("DATAFACTORY_TOOLS_PYTHON")
|
|
380
|
+
if explicit:
|
|
381
|
+
return explicit
|
|
382
|
+
# DATAFACTORY_TOOLS_HOME moves the conventional environment -- and lets
|
|
383
|
+
# the test suite point it somewhere empty, so no test ever finds the real
|
|
384
|
+
# sidecar a developer has installed and spawns PyTorch by accident.
|
|
385
|
+
candidate = Path(os.environ.get("DATAFACTORY_TOOLS_HOME") or Path.home() / ".datafactory" / "tools")
|
|
386
|
+
binary = candidate / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
|
|
387
|
+
if binary.is_file():
|
|
388
|
+
return str(binary)
|
|
389
|
+
return sys.executable
|