phytograph 0.98.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phytograph/__init__.py +27 -0
- phytograph/__main__.py +67 -0
- phytograph/_backend_manifest.json +9 -0
- phytograph/_download.py +331 -0
- phytograph/_frames.py +119 -0
- phytograph/_ports.py +45 -0
- phytograph/_transport.py +141 -0
- phytograph/_version.py +6 -0
- phytograph/backend.py +533 -0
- phytograph/cloud.py +508 -0
- phytograph/errors.py +34 -0
- phytograph/results.py +319 -0
- phytograph-0.98.0.dist-info/METADATA +41 -0
- phytograph-0.98.0.dist-info/RECORD +16 -0
- phytograph-0.98.0.dist-info/WHEEL +4 -0
- phytograph-0.98.0.dist-info/top_level.txt +1 -0
phytograph/__init__.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Phytograph scripting API.
|
|
2
|
+
|
|
3
|
+
Run Phytograph's point-cloud processing headless, from a script::
|
|
4
|
+
|
|
5
|
+
import phytograph as pg
|
|
6
|
+
|
|
7
|
+
with pg.Backend() as be: # launches a private headless backend
|
|
8
|
+
cloud = be.open("plot.laz")
|
|
9
|
+
cloud.crop([0, 0, -5], [20, 20, 40])
|
|
10
|
+
cloud.export("plot-cropped.laz")
|
|
11
|
+
xyz = cloud.positions # (N, 3) float64, world coordinates
|
|
12
|
+
|
|
13
|
+
The package is pure Python (httpx + numpy). The compute runs in the same backend
|
|
14
|
+
the desktop app ships, located automatically (see :class:`Backend`).
|
|
15
|
+
"""
|
|
16
|
+
from ._version import EXPECTED_BACKEND_VERSION, __version__
|
|
17
|
+
from .backend import Backend
|
|
18
|
+
from .cloud import Cloud
|
|
19
|
+
from .errors import (BackendNotFound, BackendStartError, Canceled, CostWarning,
|
|
20
|
+
PhytographError, VersionMismatch)
|
|
21
|
+
from .results import (DEM, GROUND, LEAF, NOISE, NOISE_CLEAN, NON_GROUND, QSM, WOOD, Grid,
|
|
22
|
+
Inventory, LADResult, Mesh, Registration, Scan, Skeleton)
|
|
23
|
+
|
|
24
|
+
__all__ = ["Backend", "Cloud", "PhytographError", "BackendNotFound", "BackendStartError",
|
|
25
|
+
"VersionMismatch", "Canceled", "CostWarning", "EXPECTED_BACKEND_VERSION", "__version__",
|
|
26
|
+
"DEM", "Mesh", "QSM", "Skeleton", "Inventory", "Registration", "LADResult", "Scan", "Grid",
|
|
27
|
+
"GROUND", "NON_GROUND", "WOOD", "LEAF", "NOISE", "NOISE_CLEAN"]
|
phytograph/__main__.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""``python -m phytograph`` — manage the backend a script will use.
|
|
2
|
+
|
|
3
|
+
python -m phytograph download # fetch + verify now (e.g. before going offline)
|
|
4
|
+
python -m phytograph download --archive F # install from an archive copied over by hand
|
|
5
|
+
python -m phytograph where # which backend `Backend()` would launch
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import sys
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from . import _download
|
|
14
|
+
from ._version import EXPECTED_BACKEND_VERSION
|
|
15
|
+
from .errors import PhytographError
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _progress(frac, msg):
|
|
19
|
+
pct = f"{frac * 100:5.1f}% " if frac is not None else ""
|
|
20
|
+
print(f"\r {pct}{msg}".ljust(78), end="", file=sys.stderr, flush=True)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def main(argv=None) -> int:
|
|
24
|
+
ap = argparse.ArgumentParser(prog="python -m phytograph", description=__doc__.splitlines()[0])
|
|
25
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
26
|
+
d = sub.add_parser("download", help="download and verify the backend for this version")
|
|
27
|
+
d.add_argument("--archive", type=Path,
|
|
28
|
+
help="install from this already-downloaded archive instead (still verified)")
|
|
29
|
+
sub.add_parser("where", help="show which backend Backend() would use")
|
|
30
|
+
args = ap.parse_args(argv)
|
|
31
|
+
try:
|
|
32
|
+
if args.cmd == "download":
|
|
33
|
+
os_name, arch = _download.platform_key()
|
|
34
|
+
name = _download.asset_name(EXPECTED_BACKEND_VERSION, os_name, arch)
|
|
35
|
+
print(f"phytograph {EXPECTED_BACKEND_VERSION}: backend {name}", file=sys.stderr)
|
|
36
|
+
exe = _download.install(archive=args.archive, progress=_progress)
|
|
37
|
+
print(file=sys.stderr)
|
|
38
|
+
print(exe)
|
|
39
|
+
else:
|
|
40
|
+
from .backend import _resolve_command
|
|
41
|
+
cached = _download.cached_executable()
|
|
42
|
+
print(f"phytograph {EXPECTED_BACKEND_VERSION}")
|
|
43
|
+
print(f"download cache: {_download.cache_root()}"
|
|
44
|
+
+ (f" (installed: {cached})" if cached else " (nothing downloaded yet)"))
|
|
45
|
+
# `auto`'s order, without ever starting a download: the explicit
|
|
46
|
+
# executable and dev checkout ("dev"), an app of this version
|
|
47
|
+
# ("app"), then the download cache.
|
|
48
|
+
desc = None
|
|
49
|
+
for src in ("dev", "app"):
|
|
50
|
+
try:
|
|
51
|
+
desc = _resolve_command(src, None)[3]
|
|
52
|
+
break
|
|
53
|
+
except PhytographError:
|
|
54
|
+
continue
|
|
55
|
+
if desc is None and cached is not None:
|
|
56
|
+
desc = f"downloaded backend {cached}"
|
|
57
|
+
print(f"Backend() would use: {desc}" if desc else
|
|
58
|
+
"Backend() would download the backend on first use "
|
|
59
|
+
"(or run: python -m phytograph download)")
|
|
60
|
+
except PhytographError as e:
|
|
61
|
+
print(f"\nerror: {e}", file=sys.stderr)
|
|
62
|
+
return 1
|
|
63
|
+
return 0
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
if __name__ == "__main__":
|
|
67
|
+
sys.exit(main())
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "0.98.0",
|
|
3
|
+
"sha256": {
|
|
4
|
+
"phytograph-backend-0.98.0-linux-x64.tar.gz": "b96a6b33afa3ce0e71752794ec1df2344bf86c4c2eb888c9059ec6f759727d3c",
|
|
5
|
+
"phytograph-backend-0.98.0-macos-arm64.tar.gz": "209bfb36da5bb125a0f29da7bd2dd0f04a00525b8c1907c6d9ec853e8841352c",
|
|
6
|
+
"phytograph-backend-0.98.0-macos-x64.tar.gz": "262caa77f03c34c8b124fd68757c49bb02f1241ba498f60d12ed2db554f2401d",
|
|
7
|
+
"phytograph-backend-0.98.0-windows-x64.zip": "ecdd69918bd7f7389071939b146c0ad2c1be59d89587342916e4479cc2f13ce2"
|
|
8
|
+
}
|
|
9
|
+
}
|
phytograph/_download.py
ADDED
|
@@ -0,0 +1,331 @@
|
|
|
1
|
+
"""Fetching the headless backend, so ``pip install phytograph`` is enough.
|
|
2
|
+
|
|
3
|
+
Every release publishes, per platform, an archive of exactly what the desktop
|
|
4
|
+
app ships under ``Contents/Resources`` (macOS) / ``resources`` (Windows, Linux),
|
|
5
|
+
minus Electron:
|
|
6
|
+
|
|
7
|
+
resources/phytograph_backend/ the PyInstaller backend
|
|
8
|
+
resources/potree_converter/ needed by every import
|
|
9
|
+
resources/ml_models/ bundled segmentation models
|
|
10
|
+
docker/riegl/ the RIEGL reader build context
|
|
11
|
+
phytograph-backend.json {version, os, arch}
|
|
12
|
+
|
|
13
|
+
The layout is the app's on purpose: ``PHYTOGRAPH_RESOURCES`` and the backend's
|
|
14
|
+
"one level up" lookups (see CLAUDE.md on the RIEGL context) then resolve exactly
|
|
15
|
+
as in a packaged app.
|
|
16
|
+
|
|
17
|
+
This module is the ONE definition of the archive's name and layout. The
|
|
18
|
+
release-side packer (``python/pack_backend.py``) imports it, and the client
|
|
19
|
+
downloads by it, so they cannot disagree.
|
|
20
|
+
|
|
21
|
+
Integrity: a wheel built by the release workflow embeds the SHA-256 of every
|
|
22
|
+
platform's archive (``_backend_manifest.json``), so a PyPI install pins the
|
|
23
|
+
exact bytes it will execute. Without that manifest (a source checkout, or the
|
|
24
|
+
wheel shipped inside the app) the digest is fetched from the release's
|
|
25
|
+
``<asset>.sha256`` over HTTPS. Either way a mismatch is refused, and nothing
|
|
26
|
+
unverified is ever extracted.
|
|
27
|
+
|
|
28
|
+
Retention: like the installers, archives are kept for the newest two releases
|
|
29
|
+
only (``scripts/prune-release-assets.mjs``), so an older pinned client finds
|
|
30
|
+
none and says how to proceed.
|
|
31
|
+
"""
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import hashlib
|
|
35
|
+
import json
|
|
36
|
+
import os
|
|
37
|
+
import platform
|
|
38
|
+
import posixpath
|
|
39
|
+
import shutil
|
|
40
|
+
import sys
|
|
41
|
+
import tarfile
|
|
42
|
+
import tempfile
|
|
43
|
+
import zipfile
|
|
44
|
+
from pathlib import Path, PurePosixPath
|
|
45
|
+
from typing import Callable, Dict, Optional, Tuple
|
|
46
|
+
|
|
47
|
+
from ._version import EXPECTED_BACKEND_VERSION
|
|
48
|
+
from .errors import BackendNotFound
|
|
49
|
+
|
|
50
|
+
REPO = "PlantSimulationLab/Phytograph"
|
|
51
|
+
DEFAULT_URL_BASE = f"https://github.com/{REPO}/releases/download"
|
|
52
|
+
# The Linux bundle's floor (scripts/glibc-floor.mjs GLIBC_FLOOR): checked before
|
|
53
|
+
# a 400 MB download that could never run here. A test pins the two equal.
|
|
54
|
+
LINUX_GLIBC_FLOOR = (2, 38)
|
|
55
|
+
MANIFEST_NAME = "phytograph-backend.json"
|
|
56
|
+
_EXE = {"windows": "phytograph_backend.exe"}
|
|
57
|
+
|
|
58
|
+
ProgressFn = Callable[[Optional[float], str], None]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# --------------------------------------------------------------------------
|
|
62
|
+
# Naming
|
|
63
|
+
# --------------------------------------------------------------------------
|
|
64
|
+
def platform_key() -> Tuple[str, str]:
|
|
65
|
+
"""(os, arch) of this machine in archive terms."""
|
|
66
|
+
if sys.platform == "darwin":
|
|
67
|
+
os_name = "macos"
|
|
68
|
+
elif sys.platform == "win32":
|
|
69
|
+
os_name = "windows"
|
|
70
|
+
elif sys.platform.startswith("linux"):
|
|
71
|
+
os_name = "linux"
|
|
72
|
+
else:
|
|
73
|
+
raise BackendNotFound(f"No Phytograph backend is built for {sys.platform}.")
|
|
74
|
+
machine = platform.machine().lower()
|
|
75
|
+
arch = {"arm64": "arm64", "aarch64": "arm64", "x86_64": "x64", "amd64": "x64"}.get(machine)
|
|
76
|
+
if arch is None:
|
|
77
|
+
raise BackendNotFound(f"No Phytograph backend is built for {machine} processors.")
|
|
78
|
+
if (os_name, arch) not in SUPPORTED:
|
|
79
|
+
raise BackendNotFound(f"No Phytograph backend is built for {os_name} on {arch} "
|
|
80
|
+
f"(published: {', '.join(f'{o}-{a}' for o, a in SUPPORTED)}).")
|
|
81
|
+
return os_name, arch
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
# The release matrix (.github/workflows/release.yml `build`).
|
|
85
|
+
SUPPORTED = (("macos", "arm64"), ("macos", "x64"), ("windows", "x64"), ("linux", "x64"))
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def asset_name(version: str, os_name: str, arch: str) -> str:
|
|
89
|
+
ext = "zip" if os_name == "windows" else "tar.gz"
|
|
90
|
+
return f"phytograph-backend-{version}-{os_name}-{arch}.{ext}"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def exe_relpath(os_name: str) -> PurePosixPath:
|
|
94
|
+
return PurePosixPath("resources", "phytograph_backend", _EXE.get(os_name, "phytograph_backend"))
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
# --------------------------------------------------------------------------
|
|
98
|
+
# Where downloads live
|
|
99
|
+
# --------------------------------------------------------------------------
|
|
100
|
+
def cache_root() -> Path:
|
|
101
|
+
"""Per-user cache: the same OS cache directory the backend's octree cache
|
|
102
|
+
uses (src/shared/octreeCacheRoot.contract.json), never a temp dir (reaped)."""
|
|
103
|
+
override = os.environ.get("PHYTOGRAPH_BACKEND_CACHE")
|
|
104
|
+
if override:
|
|
105
|
+
return Path(override).expanduser()
|
|
106
|
+
home = Path.home()
|
|
107
|
+
if sys.platform == "darwin":
|
|
108
|
+
base = home / "Library" / "Caches" / "Phytograph"
|
|
109
|
+
elif sys.platform == "win32":
|
|
110
|
+
base = Path(os.environ.get("LOCALAPPDATA") or home / "AppData" / "Local") / "Phytograph" / "cache"
|
|
111
|
+
else:
|
|
112
|
+
base = Path(os.environ.get("XDG_CACHE_HOME") or home / ".cache") / "Phytograph"
|
|
113
|
+
return base / "backend"
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def install_dir(version: str, os_name: str, arch: str) -> Path:
|
|
117
|
+
return cache_root() / f"{version}-{os_name}-{arch}"
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def cached_executable(version: str = EXPECTED_BACKEND_VERSION) -> Optional[Path]:
|
|
121
|
+
"""The executable of a complete, verified install, or None."""
|
|
122
|
+
try:
|
|
123
|
+
os_name, arch = platform_key()
|
|
124
|
+
except BackendNotFound:
|
|
125
|
+
return None
|
|
126
|
+
d = install_dir(version, os_name, arch)
|
|
127
|
+
exe = d.joinpath(*exe_relpath(os_name).parts)
|
|
128
|
+
return exe if (d / ".complete").is_file() and exe.is_file() else None
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
# --------------------------------------------------------------------------
|
|
132
|
+
# Digests
|
|
133
|
+
# --------------------------------------------------------------------------
|
|
134
|
+
def embedded_digests() -> Dict[str, str]:
|
|
135
|
+
"""{asset name: sha256} built into a release wheel; empty otherwise."""
|
|
136
|
+
p = Path(__file__).with_name("_backend_manifest.json")
|
|
137
|
+
if not p.is_file():
|
|
138
|
+
return {}
|
|
139
|
+
data = json.loads(p.read_text())
|
|
140
|
+
return dict(data.get("sha256", {}))
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def sha256_file(path: Path) -> str:
|
|
144
|
+
h = hashlib.sha256()
|
|
145
|
+
with open(path, "rb") as f:
|
|
146
|
+
for block in iter(lambda: f.read(1 << 20), b""):
|
|
147
|
+
h.update(block)
|
|
148
|
+
return h.hexdigest()
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _check_glibc(os_name: str) -> None:
|
|
152
|
+
if os_name != "linux":
|
|
153
|
+
return
|
|
154
|
+
lib, ver = platform.libc_ver()
|
|
155
|
+
if lib != "glibc" or not ver:
|
|
156
|
+
raise BackendNotFound(
|
|
157
|
+
"The Phytograph backend for Linux needs glibc "
|
|
158
|
+
f"{'.'.join(map(str, LINUX_GLIBC_FLOOR))}+, and this system's C library could not be "
|
|
159
|
+
"identified (musl-based distributions such as Alpine are not supported).")
|
|
160
|
+
have = tuple(int(x) for x in ver.split(".")[:2])
|
|
161
|
+
if have < LINUX_GLIBC_FLOOR:
|
|
162
|
+
raise BackendNotFound(
|
|
163
|
+
f"This system has glibc {ver}; the Phytograph backend needs "
|
|
164
|
+
f"{'.'.join(map(str, LINUX_GLIBC_FLOOR))} or newer (Ubuntu 23.10+/24.04, Debian 13, "
|
|
165
|
+
"Fedora 39+). Older systems, including Ubuntu 22.04 and RHEL/Rocky 8-9, are not "
|
|
166
|
+
"supported yet.")
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
# --------------------------------------------------------------------------
|
|
170
|
+
# Download + install
|
|
171
|
+
# --------------------------------------------------------------------------
|
|
172
|
+
def _http_get(url: str, dest: Optional[Path], progress: Optional[ProgressFn],
|
|
173
|
+
what: str) -> Optional[bytes]:
|
|
174
|
+
import httpx
|
|
175
|
+
try:
|
|
176
|
+
with httpx.stream("GET", url, follow_redirects=True,
|
|
177
|
+
timeout=httpx.Timeout(60.0, connect=20.0)) as r:
|
|
178
|
+
if r.status_code == 404:
|
|
179
|
+
return None
|
|
180
|
+
r.raise_for_status()
|
|
181
|
+
if dest is None:
|
|
182
|
+
return r.read()
|
|
183
|
+
total = int(r.headers.get("content-length") or 0)
|
|
184
|
+
done = 0
|
|
185
|
+
with open(dest, "wb") as f:
|
|
186
|
+
for chunk in r.iter_bytes(1 << 20):
|
|
187
|
+
f.write(chunk)
|
|
188
|
+
done += len(chunk)
|
|
189
|
+
if progress is not None:
|
|
190
|
+
progress(done / total if total else None,
|
|
191
|
+
f"Downloading {what} ({done / 1e6:.0f} of {total / 1e6:.0f} MB)"
|
|
192
|
+
if total else f"Downloading {what} ({done / 1e6:.0f} MB)")
|
|
193
|
+
return b""
|
|
194
|
+
except httpx.HTTPError as e:
|
|
195
|
+
raise BackendNotFound(f"Could not download {url}: {e}") from None
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _safe_extract(archive: Path, dest: Path) -> None:
|
|
199
|
+
"""Extract, refusing anything that would land outside `dest`: absolute
|
|
200
|
+
paths, `..`, hard links and devices, and any symlink whose target leaves
|
|
201
|
+
`dest`."""
|
|
202
|
+
root = dest.resolve()
|
|
203
|
+
|
|
204
|
+
def _ok(name: str) -> Path:
|
|
205
|
+
p = PurePosixPath(name)
|
|
206
|
+
if p.is_absolute() or ".." in p.parts or (p.parts and ":" in p.parts[0]):
|
|
207
|
+
raise BackendNotFound(f"Refusing unsafe path in backend archive: {name!r}")
|
|
208
|
+
target = (root / Path(*p.parts)).resolve()
|
|
209
|
+
if target != root and root not in target.parents:
|
|
210
|
+
raise BackendNotFound(f"Refusing unsafe path in backend archive: {name!r}")
|
|
211
|
+
return target
|
|
212
|
+
|
|
213
|
+
if archive.name.endswith(".zip"):
|
|
214
|
+
with zipfile.ZipFile(archive) as z:
|
|
215
|
+
for info in z.infolist():
|
|
216
|
+
target = _ok(info.filename)
|
|
217
|
+
if info.is_dir():
|
|
218
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
219
|
+
continue
|
|
220
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
221
|
+
with z.open(info) as src, open(target, "wb") as out:
|
|
222
|
+
shutil.copyfileobj(src, out)
|
|
223
|
+
mode = (info.external_attr >> 16) & 0o777
|
|
224
|
+
if mode:
|
|
225
|
+
os.chmod(target, mode)
|
|
226
|
+
else:
|
|
227
|
+
with tarfile.open(archive, "r:gz") as t:
|
|
228
|
+
members = t.getmembers()
|
|
229
|
+
for m in members:
|
|
230
|
+
_ok(m.name)
|
|
231
|
+
if m.issym():
|
|
232
|
+
# PyInstaller's macOS/Linux bundles carry relative dylib
|
|
233
|
+
# links (`_internal/libtorch.dylib -> torch/lib/...`). Allow
|
|
234
|
+
# those, and only those that stay inside the install.
|
|
235
|
+
if os.path.isabs(m.linkname):
|
|
236
|
+
raise BackendNotFound(f"Refusing absolute link in backend archive: {m.name!r}")
|
|
237
|
+
target = posixpath.normpath(str(PurePosixPath(m.name).parent / m.linkname))
|
|
238
|
+
if target == ".." or target.startswith("../"):
|
|
239
|
+
raise BackendNotFound(f"Refusing link out of the backend archive: {m.name!r}")
|
|
240
|
+
_ok(target)
|
|
241
|
+
elif not (m.isfile() or m.isdir()):
|
|
242
|
+
raise BackendNotFound(f"Refusing link or special file in backend archive: {m.name!r}")
|
|
243
|
+
if hasattr(tarfile, "data_filter"): # 3.12+: the stdlib's own guard too
|
|
244
|
+
t.extractall(root, members=members, filter="data")
|
|
245
|
+
else:
|
|
246
|
+
t.extractall(root, members=members)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def install(*, version: str = EXPECTED_BACKEND_VERSION, archive: Optional[Path] = None,
|
|
250
|
+
progress: Optional[ProgressFn] = None) -> Path:
|
|
251
|
+
"""Make the backend for this client's version available locally and return
|
|
252
|
+
its executable. Downloads it unless a verified copy is cached, or installs
|
|
253
|
+
from ``archive`` (a file you downloaded yourself — still verified)."""
|
|
254
|
+
os_name, arch = platform_key()
|
|
255
|
+
_check_glibc(os_name)
|
|
256
|
+
exe = cached_executable(version)
|
|
257
|
+
if exe is not None and archive is None:
|
|
258
|
+
return exe
|
|
259
|
+
|
|
260
|
+
name = asset_name(version, os_name, arch)
|
|
261
|
+
base = os.environ.get("PHYTOGRAPH_BACKEND_URL", DEFAULT_URL_BASE).rstrip("/")
|
|
262
|
+
url = f"{base}/v{version}/{name}"
|
|
263
|
+
src_archive = None
|
|
264
|
+
if archive is not None:
|
|
265
|
+
src_archive = Path(archive).expanduser()
|
|
266
|
+
if not src_archive.is_file():
|
|
267
|
+
raise BackendNotFound(f"No archive at {src_archive}.")
|
|
268
|
+
if src_archive.name != name:
|
|
269
|
+
raise BackendNotFound(f"{src_archive.name} is not this machine's backend archive; expected {name}.")
|
|
270
|
+
|
|
271
|
+
expected = embedded_digests().get(name)
|
|
272
|
+
if expected is None and src_archive is not None:
|
|
273
|
+
# Offline install: the release's `<asset>.sha256`, copied alongside.
|
|
274
|
+
side = src_archive.with_name(src_archive.name + ".sha256")
|
|
275
|
+
if side.is_file():
|
|
276
|
+
expected = side.read_text().split()[0].strip().lower()
|
|
277
|
+
if expected is None:
|
|
278
|
+
body = _http_get(url + ".sha256", None, None, "checksum")
|
|
279
|
+
if body is None:
|
|
280
|
+
if src_archive is not None:
|
|
281
|
+
raise BackendNotFound(
|
|
282
|
+
f"Cannot verify {src_archive.name}: copy {name}.sha256 from the same release "
|
|
283
|
+
"next to it (or install phytograph from PyPI, whose wheel carries the digests).")
|
|
284
|
+
raise _not_published(version, name)
|
|
285
|
+
expected = body.decode().split()[0].strip().lower()
|
|
286
|
+
|
|
287
|
+
final = install_dir(version, os_name, arch)
|
|
288
|
+
final.parent.mkdir(parents=True, exist_ok=True)
|
|
289
|
+
with tempfile.TemporaryDirectory(prefix=".incoming-", dir=final.parent) as tmp:
|
|
290
|
+
tmp = Path(tmp)
|
|
291
|
+
if src_archive is not None:
|
|
292
|
+
src = src_archive
|
|
293
|
+
else:
|
|
294
|
+
src = tmp / name
|
|
295
|
+
if _http_get(url, src, progress, f"Phytograph backend {version}") is None:
|
|
296
|
+
raise _not_published(version, name)
|
|
297
|
+
got = sha256_file(src)
|
|
298
|
+
if got != expected:
|
|
299
|
+
raise BackendNotFound(
|
|
300
|
+
f"The downloaded backend failed its integrity check ({name}: sha256 {got}, "
|
|
301
|
+
f"expected {expected}). Nothing was installed. Try again; if it persists, "
|
|
302
|
+
"report it — the file may have been altered.")
|
|
303
|
+
if progress is not None:
|
|
304
|
+
progress(None, "Unpacking the Phytograph backend")
|
|
305
|
+
staged = tmp / "staged"
|
|
306
|
+
staged.mkdir()
|
|
307
|
+
_safe_extract(src, staged)
|
|
308
|
+
meta = json.loads((staged / MANIFEST_NAME).read_text())
|
|
309
|
+
if (meta.get("version"), meta.get("os"), meta.get("arch")) != (version, os_name, arch):
|
|
310
|
+
raise BackendNotFound(f"{name} contains a backend for {meta}; refusing it.")
|
|
311
|
+
(staged / ".complete").write_text(expected + "\n")
|
|
312
|
+
if final.exists():
|
|
313
|
+
shutil.rmtree(final) # an incomplete or replaced earlier install
|
|
314
|
+
try:
|
|
315
|
+
staged.rename(final)
|
|
316
|
+
except OSError:
|
|
317
|
+
# Another process finished the same install first; use theirs.
|
|
318
|
+
if cached_executable(version) is None:
|
|
319
|
+
raise
|
|
320
|
+
exe = cached_executable(version)
|
|
321
|
+
if exe is None:
|
|
322
|
+
raise BackendNotFound(f"The backend archive {name} has no {exe_relpath(os_name)}.")
|
|
323
|
+
return exe
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _not_published(version: str, name: str) -> BackendNotFound:
|
|
327
|
+
return BackendNotFound(
|
|
328
|
+
f"The backend for phytograph {version} ({name}) is not available for download. "
|
|
329
|
+
"Backends are kept for the two newest releases only. Upgrade with "
|
|
330
|
+
"`pip install -U phytograph`, or point PHYTOGRAPH_BACKEND_EXE at the backend inside a "
|
|
331
|
+
f"Phytograph {version} app (…/resources/phytograph_backend/).")
|
phytograph/_frames.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Decoders for the backend's streaming wire format.
|
|
2
|
+
|
|
3
|
+
Long-running endpoints answer `application/octet-stream`:
|
|
4
|
+
|
|
5
|
+
* optional 4-byte whitespace keepalives,
|
|
6
|
+
* any number of `PHP1` progress markers, each ``b"PHP1" + <u32 len> + JSON``.
|
|
7
|
+
The first carries ``{"run_id": ...}`` (the handle for ``/api/cancel``); a
|
|
8
|
+
terminal one carries ``"canceled": true`` or ``"error": "..."``, because the
|
|
9
|
+
200 status line is already on the wire by the time the worker fails,
|
|
10
|
+
* then the payload: a ``PHB1`` binary frame, or a plain JSON document.
|
|
11
|
+
|
|
12
|
+
This module is the ONE Python decoder for that format. The backend's own tests
|
|
13
|
+
import it (``backend-api/tests/binframe.py``) so the client and the tests can
|
|
14
|
+
never drift apart. Mirrors the renderer's ``decodeBinaryFrame``
|
|
15
|
+
(``src/renderer/utils/backendApi.ts``).
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import struct
|
|
21
|
+
from typing import Any, Dict, Iterator, List, Optional, Tuple
|
|
22
|
+
|
|
23
|
+
import numpy as np
|
|
24
|
+
|
|
25
|
+
PROGRESS_MAGIC = b"PHP1"
|
|
26
|
+
FRAME_MAGIC = b"PHB1"
|
|
27
|
+
_WS = (0x20, 0x09, 0x0A, 0x0D)
|
|
28
|
+
_DTYPES = {"f32": np.float32, "u32": np.uint32}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class StreamParser:
|
|
32
|
+
"""Incremental parser: ``feed()`` bytes as they arrive, get back the PHP1
|
|
33
|
+
markers completed so far; once the stream ends, ``payload()`` is whatever
|
|
34
|
+
followed the last marker (a PHB1 frame or JSON bytes)."""
|
|
35
|
+
|
|
36
|
+
def __init__(self) -> None:
|
|
37
|
+
self._buf = bytearray()
|
|
38
|
+
self._pos = 0
|
|
39
|
+
self._in_markers = True
|
|
40
|
+
|
|
41
|
+
def feed(self, chunk: bytes) -> List[Dict[str, Any]]:
|
|
42
|
+
self._buf += chunk
|
|
43
|
+
out: List[Dict[str, Any]] = []
|
|
44
|
+
while self._in_markers:
|
|
45
|
+
i = self._pos
|
|
46
|
+
while i < len(self._buf) and self._buf[i] in _WS:
|
|
47
|
+
i += 1
|
|
48
|
+
if len(self._buf) - i < 4:
|
|
49
|
+
self._pos = i
|
|
50
|
+
break
|
|
51
|
+
if self._buf[i:i + 4] != PROGRESS_MAGIC:
|
|
52
|
+
self._pos = i
|
|
53
|
+
self._in_markers = False
|
|
54
|
+
break
|
|
55
|
+
if len(self._buf) - i < 8:
|
|
56
|
+
self._pos = i
|
|
57
|
+
break
|
|
58
|
+
n = struct.unpack_from("<I", self._buf, i + 4)[0]
|
|
59
|
+
if len(self._buf) - i < 8 + n:
|
|
60
|
+
self._pos = i
|
|
61
|
+
break
|
|
62
|
+
raw = bytes(self._buf[i + 8:i + 8 + n])
|
|
63
|
+
try:
|
|
64
|
+
out.append(json.loads(raw.decode("utf-8")))
|
|
65
|
+
except (ValueError, UnicodeDecodeError) as e:
|
|
66
|
+
# A corrupt marker means the stream itself is broken; skipping it
|
|
67
|
+
# could skip the terminal error/cancel marker and decode garbage.
|
|
68
|
+
raise ValueError(f"malformed PHP1 progress marker: {raw[:80]!r}") from e
|
|
69
|
+
self._pos = i + 8 + n
|
|
70
|
+
return out
|
|
71
|
+
|
|
72
|
+
def payload(self) -> bytes:
|
|
73
|
+
return bytes(self._buf[self._pos:])
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def iter_markers(content: bytes) -> Iterator[Dict[str, Any]]:
|
|
77
|
+
"""Every PHP1 marker at the head of a complete body."""
|
|
78
|
+
p = StreamParser()
|
|
79
|
+
yield from p.feed(content)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def split_stream(content: bytes) -> Tuple[List[Dict[str, Any]], bytes]:
|
|
83
|
+
"""(markers, payload) for a complete body."""
|
|
84
|
+
p = StreamParser()
|
|
85
|
+
markers = p.feed(content)
|
|
86
|
+
return markers, p.payload()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def decode_frame(payload: bytes) -> Tuple[Dict[str, Any], Dict[str, np.ndarray]]:
|
|
90
|
+
"""Decode one PHB1 frame into ``(meta, {name: array})``. Leading whitespace
|
|
91
|
+
and PHP1 markers are skipped, so a whole streamed body is accepted too."""
|
|
92
|
+
_, payload = split_stream(payload)
|
|
93
|
+
i = 0
|
|
94
|
+
while i < len(payload) and payload[i] in _WS:
|
|
95
|
+
i += 1
|
|
96
|
+
if payload[i:i + 4] != FRAME_MAGIC:
|
|
97
|
+
raise ValueError("not a PHB1 frame")
|
|
98
|
+
header_len = struct.unpack_from("<I", payload, i + 4)[0]
|
|
99
|
+
header = json.loads(payload[i + 8:i + 8 + header_len].decode("utf-8"))
|
|
100
|
+
off = i + 8 + header_len
|
|
101
|
+
buffers: Dict[str, np.ndarray] = {}
|
|
102
|
+
for d in header["buffers"]:
|
|
103
|
+
dtype = _DTYPES[d["dtype"]]
|
|
104
|
+
n = int(d["length"])
|
|
105
|
+
buffers[d["name"]] = np.frombuffer(payload, dtype=dtype, count=n, offset=off).copy()
|
|
106
|
+
off += n * 4
|
|
107
|
+
return header["meta"], buffers
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def is_frame(payload: bytes) -> bool:
|
|
111
|
+
return payload.lstrip(b" \t\r\n")[:4] == FRAME_MAGIC
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def terminal_marker(markers: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
|
115
|
+
"""The in-band failure/cancel marker, if the stream ended with one."""
|
|
116
|
+
for m in reversed(markers):
|
|
117
|
+
if isinstance(m, dict) and (m.get("error") or m.get("canceled")):
|
|
118
|
+
return m
|
|
119
|
+
return None
|
phytograph/_ports.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Free-port choice for a script-launched backend.
|
|
2
|
+
|
|
3
|
+
The Python twin of ``scripts/free-port.mjs`` — read that file's header for the
|
|
4
|
+
reasoning. In short: never bind ``:0`` (an ephemeral port can be handed to an
|
|
5
|
+
outgoing connection between our probe and the server's bind), and draw from a
|
|
6
|
+
band of our own below every OS ephemeral range so a script, the desktop app,
|
|
7
|
+
``npm run dev`` and E2E never consider the same number.
|
|
8
|
+
|
|
9
|
+
``SCRIPT_BAND`` is the ``script`` entry of ``PORT_BANDS`` in
|
|
10
|
+
``scripts/free-port.mjs``; ``scripts/free-port.test.mjs`` reads this file and
|
|
11
|
+
asserts the two agree (and that no band overlaps another).
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import random
|
|
16
|
+
import socket
|
|
17
|
+
from typing import Iterable, Tuple
|
|
18
|
+
|
|
19
|
+
# [min, max). The other owners: app 10000-14999, dev 15000-19999, e2e 20000-31999.
|
|
20
|
+
SCRIPT_BAND: Tuple[int, int] = (5_000, 10_000)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def can_listen(port: int, host: str = "127.0.0.1") -> bool:
|
|
24
|
+
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
25
|
+
try:
|
|
26
|
+
s.bind((host, port))
|
|
27
|
+
s.listen(1)
|
|
28
|
+
return True
|
|
29
|
+
except OSError:
|
|
30
|
+
return False
|
|
31
|
+
finally:
|
|
32
|
+
s.close()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def find_free_port(band: Tuple[int, int] = SCRIPT_BAND, *, exclude: Iterable[int] = (),
|
|
36
|
+
tries: int = 200, host: str = "127.0.0.1") -> int:
|
|
37
|
+
lo, hi = band
|
|
38
|
+
skip = set(exclude)
|
|
39
|
+
for _ in range(tries):
|
|
40
|
+
port = random.randrange(lo, hi)
|
|
41
|
+
if port in skip:
|
|
42
|
+
continue
|
|
43
|
+
if can_listen(port, host):
|
|
44
|
+
return port
|
|
45
|
+
raise RuntimeError(f"no free port in [{lo}, {hi}) after {tries} tries")
|