lunar-toolkit 0.1.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lunar_toolkit/__init__.py +6 -0
- lunar_toolkit/asp.py +432 -0
- lunar_toolkit/cli/__init__.py +88 -0
- lunar_toolkit/cli/_jobs.py +437 -0
- lunar_toolkit/cli/_status.py +368 -0
- lunar_toolkit/cli/_utils.py +128 -0
- lunar_toolkit/cli/app.py +205 -0
- lunar_toolkit/cli/build_db.py +130 -0
- lunar_toolkit/cli/download.py +111 -0
- lunar_toolkit/cli/kaguya.py +253 -0
- lunar_toolkit/cli/ode_search.py +144 -0
- lunar_toolkit/cli/pairs.py +236 -0
- lunar_toolkit/cli/process.py +138 -0
- lunar_toolkit/cli/query.py +70 -0
- lunar_toolkit/cli/reference.py +233 -0
- lunar_toolkit/cli/search.py +90 -0
- lunar_toolkit/cli/serve.py +706 -0
- lunar_toolkit/cli/sfs.py +344 -0
- lunar_toolkit/cli/sfs_report.py +148 -0
- lunar_toolkit/cli/shadowcam.py +246 -0
- lunar_toolkit/cli/stereo.py +85 -0
- lunar_toolkit/cli/workspace.py +84 -0
- lunar_toolkit/config.py +345 -0
- lunar_toolkit/constants.py +217 -0
- lunar_toolkit/csm.py +340 -0
- lunar_toolkit/downloader.py +447 -0
- lunar_toolkit/geometry.py +556 -0
- lunar_toolkit/isis.py +222 -0
- lunar_toolkit/kaguya/__init__.py +45 -0
- lunar_toolkit/kaguya/mi.py +274 -0
- lunar_toolkit/kaguya/tc.py +370 -0
- lunar_toolkit/lroc/__init__.py +6 -0
- lunar_toolkit/lroc/cumindex.py +1197 -0
- lunar_toolkit/lroc/nac.py +702 -0
- lunar_toolkit/observation.py +127 -0
- lunar_toolkit/ode.py +485 -0
- lunar_toolkit/provenance.py +312 -0
- lunar_toolkit/reference.py +1111 -0
- lunar_toolkit/selection.py +714 -0
- lunar_toolkit/sfs.py +1534 -0
- lunar_toolkit/sfs_products.py +676 -0
- lunar_toolkit/sfs_report.py +1328 -0
- lunar_toolkit/shadowcam/__init__.py +29 -0
- lunar_toolkit/shadowcam/sc.py +392 -0
- lunar_toolkit/static/index.html +3259 -0
- lunar_toolkit/stereo.py +519 -0
- lunar_toolkit/stereo_dem.py +145 -0
- lunar_toolkit/tools.py +264 -0
- lunar_toolkit/workspace.py +1173 -0
- lunar_toolkit-0.1.0.dev2.dist-info/METADATA +177 -0
- lunar_toolkit-0.1.0.dev2.dist-info/RECORD +55 -0
- lunar_toolkit-0.1.0.dev2.dist-info/WHEEL +5 -0
- lunar_toolkit-0.1.0.dev2.dist-info/entry_points.txt +2 -0
- lunar_toolkit-0.1.0.dev2.dist-info/licenses/LICENSE +28 -0
- lunar_toolkit-0.1.0.dev2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,706 @@
|
|
|
1
|
+
"""serve - the web app, Range-capable Parquet / image serving, a PDS image relay, status and jobs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import hmac
|
|
7
|
+
import http.server
|
|
8
|
+
import ipaddress
|
|
9
|
+
import json
|
|
10
|
+
import math
|
|
11
|
+
import secrets
|
|
12
|
+
import shutil
|
|
13
|
+
import socket
|
|
14
|
+
import threading
|
|
15
|
+
import time
|
|
16
|
+
import urllib.error
|
|
17
|
+
import urllib.parse
|
|
18
|
+
import urllib.request
|
|
19
|
+
import webbrowser
|
|
20
|
+
from dataclasses import dataclass
|
|
21
|
+
from http.client import HTTPMessage
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import IO, Any
|
|
24
|
+
|
|
25
|
+
from ._jobs import TERMINAL, JobContext, JobManager, submit
|
|
26
|
+
from ._status import StatusCache, StatusDirs
|
|
27
|
+
|
|
28
|
+
COMMAND = "serve"
|
|
29
|
+
|
|
30
|
+
_STATIC_DIR = Path(__file__).parent.parent / "static"
|
|
31
|
+
_INDEX = _STATIC_DIR / "index.html"
|
|
32
|
+
|
|
33
|
+
_IMAGE_TYPES = {
|
|
34
|
+
".tif": "image/tiff",
|
|
35
|
+
".tiff": "image/tiff",
|
|
36
|
+
".png": "image/png",
|
|
37
|
+
".jpg": "image/jpeg",
|
|
38
|
+
".jpeg": "image/jpeg",
|
|
39
|
+
}
|
|
40
|
+
# Image hosts the page may fetch through /remote: pds.mcp.nasa.gov drops its CORS header
|
|
41
|
+
# whenever the request carries extra headers (browsers always send `priority`).
|
|
42
|
+
_RELAY_HOSTS = (
|
|
43
|
+
"pds.mcp.nasa.gov",
|
|
44
|
+
"pds.lroc.im-ldi.com",
|
|
45
|
+
"pds.shadowcam.im-ldi.com",
|
|
46
|
+
"data.darts.isas.jaxa.jp",
|
|
47
|
+
)
|
|
48
|
+
_MAX_LISTED_IMAGES = 2000
|
|
49
|
+
RELAY_MAX = 6
|
|
50
|
+
RELAY_WAIT = 30.0
|
|
51
|
+
MAX_BODY = 64 * 1024
|
|
52
|
+
SSE_KEEPALIVE = 15.0
|
|
53
|
+
SSE_MIN_INTERVAL = 0.25
|
|
54
|
+
_LOCAL_NAMES = ("localhost", "127.0.0.1", "::1")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def add_server_args(p: argparse.ArgumentParser, host: str, port: int) -> None:
|
|
58
|
+
"""Options shared by ``lunar serve`` and ``lunar app``."""
|
|
59
|
+
p.add_argument(
|
|
60
|
+
"--port",
|
|
61
|
+
type=int,
|
|
62
|
+
default=port,
|
|
63
|
+
metavar="PORT",
|
|
64
|
+
help=f"Port to listen on (default: {port or 'any free'}).",
|
|
65
|
+
)
|
|
66
|
+
p.add_argument(
|
|
67
|
+
"--host",
|
|
68
|
+
default=host,
|
|
69
|
+
metavar="HOST",
|
|
70
|
+
help=f"Address to bind (default: {host}); 0.0.0.0 serves the network (read-only there).",
|
|
71
|
+
)
|
|
72
|
+
p.add_argument(
|
|
73
|
+
"--data-dir",
|
|
74
|
+
default=None,
|
|
75
|
+
metavar="DIR",
|
|
76
|
+
help="Directory containing parquet files. Defaults to LUNAR_DATA_DIR or the user data dir.",
|
|
77
|
+
)
|
|
78
|
+
p.add_argument(
|
|
79
|
+
"--images",
|
|
80
|
+
default=None,
|
|
81
|
+
metavar="DIR",
|
|
82
|
+
help="Also serve the GeoTIFF / PNG / JPEG files under DIR (with Range) for the Image view.",
|
|
83
|
+
)
|
|
84
|
+
p.add_argument(
|
|
85
|
+
"--runs-dir",
|
|
86
|
+
default=None,
|
|
87
|
+
metavar="DIR",
|
|
88
|
+
help="SfS runs listed in the Settings tab (default: LUNAR_RUNS_DIR).",
|
|
89
|
+
)
|
|
90
|
+
p.add_argument(
|
|
91
|
+
"--admin-token",
|
|
92
|
+
default=None,
|
|
93
|
+
metavar="TOKEN",
|
|
94
|
+
help="Let remote clients with this token (Authorization: Bearer TOKEN or ?token=TOKEN) "
|
|
95
|
+
"start jobs and see local paths; 'auto' generates one.",
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def add_parser(sub: argparse._SubParsersAction) -> None:
|
|
100
|
+
p = sub.add_parser(
|
|
101
|
+
COMMAND,
|
|
102
|
+
help="Serve the Lunar Toolkit web app with local parquet files.",
|
|
103
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
104
|
+
epilog="""
|
|
105
|
+
Examples
|
|
106
|
+
--------
|
|
107
|
+
lunar serve
|
|
108
|
+
lunar serve --port 9000
|
|
109
|
+
lunar serve --data-dir /mnt/lroc-data
|
|
110
|
+
lunar serve --images ~/sfs_runs # browse local GeoTIFF / COG results in the Image view
|
|
111
|
+
lunar serve --host 0.0.0.0 # share on the network; remote clients are read-only
|
|
112
|
+
lunar serve --host 0.0.0.0 --admin-token auto # ... and print a token for remote admin
|
|
113
|
+
|
|
114
|
+
Admin (the Settings tab: build indexes, download, see local paths) is allowed from this
|
|
115
|
+
machine (loopback) and for requests carrying --admin-token; everyone else can browse,
|
|
116
|
+
search and view. Use `lunar app` for a desktop window.
|
|
117
|
+
""",
|
|
118
|
+
)
|
|
119
|
+
add_server_args(p, "localhost", 8080)
|
|
120
|
+
p.add_argument("--no-browser", action="store_true", help="Don't open a browser tab automatically.")
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _byte_range(header: str, size: int) -> tuple[int, int] | tuple[()] | None:
|
|
124
|
+
"""``(start, length)`` for a single ``bytes=`` Range, ``()`` without one, None if unsatisfiable."""
|
|
125
|
+
if not header.startswith("bytes="):
|
|
126
|
+
return ()
|
|
127
|
+
first, _, last = header[6:].split(",")[0].strip().partition("-")
|
|
128
|
+
try:
|
|
129
|
+
if not first: # suffix range: the last N bytes
|
|
130
|
+
n = int(last)
|
|
131
|
+
start, end = max(size - n, 0), size - 1
|
|
132
|
+
else:
|
|
133
|
+
start = int(first)
|
|
134
|
+
end = min(int(last), size - 1) if last else size - 1
|
|
135
|
+
except ValueError:
|
|
136
|
+
return None
|
|
137
|
+
if start >= size or end < start:
|
|
138
|
+
return None
|
|
139
|
+
return start, end - start + 1
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _image_path(root: Path, rel: str) -> Path | None:
|
|
143
|
+
"""The image file ``rel`` (URL-quoted, relative) under ``root``, or None if it escapes or isn't one."""
|
|
144
|
+
parts = urllib.parse.unquote(rel).replace("\\", "/").split("/")
|
|
145
|
+
if not rel or any(not part or part.startswith(".") for part in parts):
|
|
146
|
+
return None
|
|
147
|
+
base = root.resolve()
|
|
148
|
+
path = base.joinpath(*parts).resolve()
|
|
149
|
+
if not path.is_relative_to(base) or path.suffix.lower() not in _IMAGE_TYPES or not path.is_file():
|
|
150
|
+
return None
|
|
151
|
+
return path
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _list_images(root: Path) -> list[str]:
|
|
155
|
+
"""Relative POSIX paths of the image files under ``root`` (hidden entries skipped)."""
|
|
156
|
+
base = root.resolve()
|
|
157
|
+
out = []
|
|
158
|
+
for path in sorted(base.rglob("*")):
|
|
159
|
+
rel = path.relative_to(base)
|
|
160
|
+
if any(part.startswith(".") for part in rel.parts) or path.suffix.lower() not in _IMAGE_TYPES:
|
|
161
|
+
continue
|
|
162
|
+
if path.is_file() and path.resolve().is_relative_to(base):
|
|
163
|
+
out.append(rel.as_posix())
|
|
164
|
+
if len(out) >= _MAX_LISTED_IMAGES:
|
|
165
|
+
break
|
|
166
|
+
return out
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _report_path(runs_dir: Path, name: str) -> Path | None:
|
|
170
|
+
"""``<runs_dir>/<name>/report/report.html`` if *name* is a plain run dir name and it exists."""
|
|
171
|
+
name = urllib.parse.unquote(name)
|
|
172
|
+
if not name or "/" in name or "\\" in name or name.startswith("."):
|
|
173
|
+
return None
|
|
174
|
+
base = runs_dir.resolve()
|
|
175
|
+
path = (base / name / "report" / "report.html").resolve()
|
|
176
|
+
return path if path.is_relative_to(base) and path.is_file() else None
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _relay_allowed(url: str) -> bool:
|
|
180
|
+
"""Only https image files on the PDS / JAXA image hosts are relayed."""
|
|
181
|
+
try:
|
|
182
|
+
u = urllib.parse.urlsplit(url)
|
|
183
|
+
port = u.port
|
|
184
|
+
except ValueError:
|
|
185
|
+
return False
|
|
186
|
+
return (
|
|
187
|
+
u.scheme == "https"
|
|
188
|
+
and u.hostname in _RELAY_HOSTS
|
|
189
|
+
and port is None
|
|
190
|
+
and not u.username
|
|
191
|
+
and Path(u.path).suffix.lower() in _IMAGE_TYPES
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
class _RelayRedirect(urllib.request.HTTPRedirectHandler):
|
|
196
|
+
def redirect_request(
|
|
197
|
+
self,
|
|
198
|
+
req: urllib.request.Request,
|
|
199
|
+
fp: IO[bytes],
|
|
200
|
+
code: int,
|
|
201
|
+
msg: str,
|
|
202
|
+
headers: HTTPMessage,
|
|
203
|
+
newurl: str,
|
|
204
|
+
) -> urllib.request.Request | None:
|
|
205
|
+
if not _relay_allowed(newurl):
|
|
206
|
+
raise urllib.error.HTTPError(newurl, 403, "redirect outside the relay hosts", headers, fp)
|
|
207
|
+
return super().redirect_request(req, fp, code, msg, headers, newurl)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
_relay_opener = urllib.request.build_opener(_RelayRedirect)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def is_loopback(addr: str) -> bool:
|
|
214
|
+
"""Whether client address *addr* is this machine (IPv4, IPv6 or IPv4-mapped IPv6)."""
|
|
215
|
+
try:
|
|
216
|
+
ip = ipaddress.ip_address(addr.split("%", 1)[0])
|
|
217
|
+
except ValueError:
|
|
218
|
+
return False
|
|
219
|
+
if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
|
|
220
|
+
ip = ip.ipv4_mapped
|
|
221
|
+
return ip.is_loopback
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _local_host_header(host: str) -> bool:
|
|
225
|
+
"""A ``Host`` header naming this machine (guards loopback admin against DNS rebinding)."""
|
|
226
|
+
try:
|
|
227
|
+
name = urllib.parse.urlsplit("//" + host).hostname or ""
|
|
228
|
+
except ValueError:
|
|
229
|
+
return False
|
|
230
|
+
return name in _LOCAL_NAMES or name.endswith(".localhost")
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
@dataclass
|
|
234
|
+
class ServerConfig:
|
|
235
|
+
"""What a :class:`LunarServer` serves and where its jobs write."""
|
|
236
|
+
|
|
237
|
+
data_dir: Path
|
|
238
|
+
explicit_dir: Path | None = None
|
|
239
|
+
images_dir: Path | None = None
|
|
240
|
+
runs_dir: Path | None = None
|
|
241
|
+
admin_token: str | None = None
|
|
242
|
+
cache_dir: Path | None = None
|
|
243
|
+
download_dir: Path | None = None
|
|
244
|
+
reference_dir: Path | None = None
|
|
245
|
+
|
|
246
|
+
def __post_init__(self) -> None:
|
|
247
|
+
from ..constants import DEFAULT_CACHE_DIR
|
|
248
|
+
from ..reference import REFERENCE_CACHE_DIR
|
|
249
|
+
|
|
250
|
+
self.cache_dir = self.cache_dir or DEFAULT_CACHE_DIR
|
|
251
|
+
self.download_dir = self.download_dir or self.cache_dir / "downloads"
|
|
252
|
+
self.reference_dir = self.reference_dir or REFERENCE_CACHE_DIR
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
class LunarServer(http.server.ThreadingHTTPServer):
|
|
256
|
+
"""The HTTP server with its job manager, status cache and relay limit."""
|
|
257
|
+
|
|
258
|
+
daemon_threads = True
|
|
259
|
+
request_queue_size = 128
|
|
260
|
+
|
|
261
|
+
def __init__(self, address: tuple[str, int], cfg: ServerConfig, workers: int = 2) -> None:
|
|
262
|
+
self.cfg = cfg
|
|
263
|
+
assert cfg.cache_dir and cfg.download_dir and cfg.reference_dir
|
|
264
|
+
dirs = StatusDirs(cfg.data_dir, cfg.cache_dir, cfg.download_dir, cfg.reference_dir, cfg.runs_dir)
|
|
265
|
+
self.status = StatusCache(dirs)
|
|
266
|
+
self.jobs = JobManager(workers, on_finish=lambda job: self.status.invalidate())
|
|
267
|
+
self.job_context = JobContext(cfg.data_dir, cfg.download_dir, cfg.reference_dir)
|
|
268
|
+
self.relay_slots = threading.BoundedSemaphore(RELAY_MAX)
|
|
269
|
+
self.sdem_members: dict[int, list[dict[str, Any]]] = {}
|
|
270
|
+
if ":" in address[0]:
|
|
271
|
+
self.address_family = socket.AF_INET6
|
|
272
|
+
super().__init__(address, LunarHandler)
|
|
273
|
+
|
|
274
|
+
def close(self) -> None:
|
|
275
|
+
"""Cancel jobs, stop serving and close the socket."""
|
|
276
|
+
self.jobs.close()
|
|
277
|
+
self.shutdown()
|
|
278
|
+
self.server_close()
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
class LunarHandler(http.server.BaseHTTPRequestHandler):
|
|
282
|
+
# Keep-alive: DuckDB-WASM and the image viewer issue thousands of small Range requests
|
|
283
|
+
protocol_version = "HTTP/1.1"
|
|
284
|
+
server: LunarServer
|
|
285
|
+
|
|
286
|
+
def client_ip(self) -> str:
|
|
287
|
+
return self.client_address[0]
|
|
288
|
+
|
|
289
|
+
def _query(self) -> dict[str, list[str]]:
|
|
290
|
+
return urllib.parse.parse_qs(self.path.partition("?")[2])
|
|
291
|
+
|
|
292
|
+
def _token(self) -> str | None:
|
|
293
|
+
auth = self.headers.get("Authorization", "")
|
|
294
|
+
if auth.lower().startswith("bearer "):
|
|
295
|
+
return auth[7:].strip()
|
|
296
|
+
return self._query().get("token", [None])[0]
|
|
297
|
+
|
|
298
|
+
def is_admin(self) -> bool:
|
|
299
|
+
"""Loopback clients (with a local ``Host``) and holders of the admin token."""
|
|
300
|
+
token, given = self.server.cfg.admin_token, self._token()
|
|
301
|
+
if token and given and hmac.compare_digest(given.encode(), token.encode()):
|
|
302
|
+
return True
|
|
303
|
+
return is_loopback(self.client_ip()) and _local_host_header(self.headers.get("Host", ""))
|
|
304
|
+
|
|
305
|
+
def do_GET(self) -> None:
|
|
306
|
+
self._route(body=True)
|
|
307
|
+
|
|
308
|
+
def do_HEAD(self) -> None:
|
|
309
|
+
self._route(body=False)
|
|
310
|
+
|
|
311
|
+
def do_POST(self) -> None:
|
|
312
|
+
path = self.path.partition("?")[0]
|
|
313
|
+
if path != "/api/jobs":
|
|
314
|
+
self._error(404, "not found")
|
|
315
|
+
return
|
|
316
|
+
if not self._check_admin(write=True):
|
|
317
|
+
return
|
|
318
|
+
if "application/json" not in self.headers.get("Content-Type", ""):
|
|
319
|
+
self._error(415, "expected application/json")
|
|
320
|
+
return
|
|
321
|
+
try:
|
|
322
|
+
n = int(self.headers.get("Content-Length", "0"))
|
|
323
|
+
except ValueError:
|
|
324
|
+
n = -1
|
|
325
|
+
if not 0 < n <= MAX_BODY:
|
|
326
|
+
self._error(400 if n <= 0 else 413, "bad request body")
|
|
327
|
+
return
|
|
328
|
+
try:
|
|
329
|
+
payload = json.loads(self.rfile.read(n))
|
|
330
|
+
job = submit(self.server.jobs, payload, self.server.job_context)
|
|
331
|
+
except ValueError as exc: # also json.JSONDecodeError
|
|
332
|
+
self._error(409 if "already" in str(exc) else 400, str(exc))
|
|
333
|
+
return
|
|
334
|
+
except RuntimeError as exc:
|
|
335
|
+
self._error(503, str(exc))
|
|
336
|
+
return
|
|
337
|
+
self._json(job.to_dict(), code=201)
|
|
338
|
+
|
|
339
|
+
def do_DELETE(self) -> None:
|
|
340
|
+
path = self.path.partition("?")[0]
|
|
341
|
+
parts = path.strip("/").split("/")
|
|
342
|
+
if len(parts) != 3 or parts[:2] != ["api", "jobs"]:
|
|
343
|
+
self._error(404, "not found")
|
|
344
|
+
return
|
|
345
|
+
if not self._check_admin(write=True):
|
|
346
|
+
return
|
|
347
|
+
job = self.server.jobs.cancel(parts[2])
|
|
348
|
+
if job is None:
|
|
349
|
+
self._error(404, "no such job")
|
|
350
|
+
return
|
|
351
|
+
self._json(job.to_dict())
|
|
352
|
+
|
|
353
|
+
def do_OPTIONS(self) -> None:
|
|
354
|
+
self._error(405, "method not allowed")
|
|
355
|
+
|
|
356
|
+
def _check_admin(self, write: bool = False) -> bool:
|
|
357
|
+
if not self.is_admin():
|
|
358
|
+
self._error(403, "read-only client: admin needs localhost or the --admin-token")
|
|
359
|
+
return False
|
|
360
|
+
origin = self.headers.get("Origin")
|
|
361
|
+
if write and origin and urllib.parse.urlsplit(origin).netloc != self.headers.get("Host", ""):
|
|
362
|
+
self._error(403, "cross-origin request")
|
|
363
|
+
return False
|
|
364
|
+
return True
|
|
365
|
+
|
|
366
|
+
def _route(self, body: bool) -> None:
|
|
367
|
+
path, _, query = self.path.partition("?")
|
|
368
|
+
cfg = self.server.cfg
|
|
369
|
+
if path in ("/", "/index.html"):
|
|
370
|
+
self._serve(_INDEX, "text/html; charset=utf-8", body)
|
|
371
|
+
elif path == "/api/config":
|
|
372
|
+
from ..constants import APP_VERSION
|
|
373
|
+
|
|
374
|
+
self._json(
|
|
375
|
+
{
|
|
376
|
+
"relay": list(_RELAY_HOSTS),
|
|
377
|
+
"images": cfg.images_dir is not None,
|
|
378
|
+
"admin": self.is_admin(),
|
|
379
|
+
"jobs": True,
|
|
380
|
+
"runs": cfg.runs_dir is not None,
|
|
381
|
+
"version": APP_VERSION,
|
|
382
|
+
},
|
|
383
|
+
body,
|
|
384
|
+
)
|
|
385
|
+
elif path == "/api/images":
|
|
386
|
+
self._json(_list_images(cfg.images_dir) if cfg.images_dir else [], body)
|
|
387
|
+
elif path.startswith("/images/") and cfg.images_dir is not None:
|
|
388
|
+
img = _image_path(cfg.images_dir, path[len("/images/") :])
|
|
389
|
+
if img is None:
|
|
390
|
+
self._send_code(404)
|
|
391
|
+
return
|
|
392
|
+
self._serve(img, _IMAGE_TYPES[img.suffix.lower()], body)
|
|
393
|
+
elif path == "/remote":
|
|
394
|
+
self._relay(urllib.parse.parse_qs(query).get("url", [""])[0], body)
|
|
395
|
+
elif path.startswith("/data/"):
|
|
396
|
+
from ..constants import find_parquet
|
|
397
|
+
|
|
398
|
+
name = path[len("/data/") :]
|
|
399
|
+
if not name or "/" in name or name.startswith("."):
|
|
400
|
+
self._send_code(404)
|
|
401
|
+
return
|
|
402
|
+
self._serve(find_parquet(name, cfg.data_dir), "application/octet-stream", body)
|
|
403
|
+
elif path == "/api/status":
|
|
404
|
+
refresh = self._query().get("refresh", ["0"])[0] == "1"
|
|
405
|
+
self._json(self.server.status.get(self.is_admin(), refresh=refresh), body)
|
|
406
|
+
elif path == "/api/regions":
|
|
407
|
+
self._json(_regions(), body)
|
|
408
|
+
elif path == "/api/sdem/members":
|
|
409
|
+
if self._check_admin():
|
|
410
|
+
self._sdem_members(self._query().get("region", [""])[0], body)
|
|
411
|
+
elif path.startswith("/api/jobs"):
|
|
412
|
+
self._jobs_get(path, body)
|
|
413
|
+
elif path.startswith("/runs/") and path.endswith("/report.html") and cfg.runs_dir is not None:
|
|
414
|
+
if not self._check_admin():
|
|
415
|
+
return
|
|
416
|
+
report = _report_path(cfg.runs_dir, path[len("/runs/") : -len("/report.html")])
|
|
417
|
+
if report is None:
|
|
418
|
+
self._send_code(404)
|
|
419
|
+
return
|
|
420
|
+
self._serve(report, "text/html; charset=utf-8", body)
|
|
421
|
+
else:
|
|
422
|
+
self._send_code(404)
|
|
423
|
+
|
|
424
|
+
def _jobs_get(self, path: str, body: bool) -> None:
|
|
425
|
+
admin = self.is_admin()
|
|
426
|
+
parts = path.strip("/").split("/")[2:]
|
|
427
|
+
jobs = self.server.jobs
|
|
428
|
+
if not parts:
|
|
429
|
+
self._json([j.to_dict(admin) for j in jobs.jobs()], body)
|
|
430
|
+
elif parts == ["events"]:
|
|
431
|
+
self._events(None, admin)
|
|
432
|
+
elif len(parts) in (1, 2) and parts[1:] in ([], ["events"]):
|
|
433
|
+
job = jobs.get(parts[0])
|
|
434
|
+
if job is None:
|
|
435
|
+
self._error(404, "no such job")
|
|
436
|
+
elif len(parts) == 2:
|
|
437
|
+
self._events(parts[0], admin)
|
|
438
|
+
else:
|
|
439
|
+
self._json(job.to_dict(admin), body)
|
|
440
|
+
else:
|
|
441
|
+
self._error(404, "not found")
|
|
442
|
+
|
|
443
|
+
def _events(self, jid: str | None, admin: bool) -> None:
|
|
444
|
+
"""Server-Sent Events: ``event: jobs`` (all jobs) or ``event: job`` (one) on every change."""
|
|
445
|
+
jobs = self.server.jobs
|
|
446
|
+
self.send_response(200)
|
|
447
|
+
self._common_headers("text/event-stream")
|
|
448
|
+
self.send_header("Cache-Control", "no-cache")
|
|
449
|
+
self.end_headers()
|
|
450
|
+
self.close_connection = True
|
|
451
|
+
version = -1
|
|
452
|
+
data: Any
|
|
453
|
+
try:
|
|
454
|
+
while not jobs.closed:
|
|
455
|
+
new = jobs.wait(version, SSE_KEEPALIVE)
|
|
456
|
+
if new == version:
|
|
457
|
+
self.wfile.write(b": keepalive\n\n")
|
|
458
|
+
self.wfile.flush()
|
|
459
|
+
continue
|
|
460
|
+
version = new
|
|
461
|
+
if jid is None:
|
|
462
|
+
event, data = "jobs", [j.to_dict(admin) for j in jobs.jobs()]
|
|
463
|
+
final = False
|
|
464
|
+
else:
|
|
465
|
+
job = jobs.get(jid)
|
|
466
|
+
if job is None:
|
|
467
|
+
break
|
|
468
|
+
event, data, final = "job", job.to_dict(admin), job.status in TERMINAL
|
|
469
|
+
self.wfile.write(f"event: {event}\ndata: {json.dumps(data)}\n\n".encode())
|
|
470
|
+
self.wfile.flush()
|
|
471
|
+
if final:
|
|
472
|
+
break
|
|
473
|
+
time.sleep(SSE_MIN_INTERVAL)
|
|
474
|
+
except (BrokenPipeError, ConnectionResetError, OSError):
|
|
475
|
+
pass
|
|
476
|
+
|
|
477
|
+
def _sdem_members(self, region: str, body: bool) -> None:
|
|
478
|
+
from .. import reference as ref
|
|
479
|
+
|
|
480
|
+
try:
|
|
481
|
+
reg = ref.get_region(region)
|
|
482
|
+
except KeyError as exc:
|
|
483
|
+
self._error(400, str(exc.args[0] if exc.args else exc))
|
|
484
|
+
return
|
|
485
|
+
cache = self.server.sdem_members
|
|
486
|
+
if reg.id not in cache:
|
|
487
|
+
try:
|
|
488
|
+
df = ref.sdem_members(reg)
|
|
489
|
+
except OSError as exc:
|
|
490
|
+
self._error(502, f"could not read the zip directory: {exc}")
|
|
491
|
+
return
|
|
492
|
+
cache[reg.id] = [{"name": str(r["name"]), "size": int(r["size"])} for r in df.to_dict("records")]
|
|
493
|
+
self._json({"region": reg.id, "zip": reg.zenodo_file, "members": cache[reg.id]}, body)
|
|
494
|
+
|
|
495
|
+
def _serve(self, path: Path, ctype: str, body: bool = True) -> None:
|
|
496
|
+
if not path.exists():
|
|
497
|
+
self._send_code(404)
|
|
498
|
+
return
|
|
499
|
+
size = path.stat().st_size
|
|
500
|
+
span = _byte_range(self.headers.get("Range", ""), size)
|
|
501
|
+
if span is None:
|
|
502
|
+
self._send_code(416)
|
|
503
|
+
return
|
|
504
|
+
start, length = span if span else (0, size)
|
|
505
|
+
self.send_response(206 if span else 200)
|
|
506
|
+
self._common_headers(ctype)
|
|
507
|
+
self.send_header("Content-Length", str(length))
|
|
508
|
+
if span:
|
|
509
|
+
self.send_header("Content-Range", f"bytes {start}-{start + length - 1}/{size}")
|
|
510
|
+
self.end_headers()
|
|
511
|
+
if body:
|
|
512
|
+
self._send_bytes(path, start, length)
|
|
513
|
+
|
|
514
|
+
def _json(self, obj: object, body: bool = True, code: int = 200) -> None:
|
|
515
|
+
data = json.dumps(obj).encode()
|
|
516
|
+
self.send_response(code)
|
|
517
|
+
self._common_headers("application/json")
|
|
518
|
+
self.send_header("Cache-Control", "no-store")
|
|
519
|
+
self.send_header("Content-Length", str(len(data)))
|
|
520
|
+
self.end_headers()
|
|
521
|
+
if body:
|
|
522
|
+
self.wfile.write(data)
|
|
523
|
+
|
|
524
|
+
def _error(self, code: int, message: str) -> None:
|
|
525
|
+
self._json({"error": message}, self.command != "HEAD", code)
|
|
526
|
+
|
|
527
|
+
def _relay(self, url: str, body: bool) -> None:
|
|
528
|
+
"""Forward one Range request, through the downloader's per-host slots, pacing and cooldown."""
|
|
529
|
+
from .. import downloader
|
|
530
|
+
|
|
531
|
+
if not _relay_allowed(url):
|
|
532
|
+
self._send_code(403)
|
|
533
|
+
return
|
|
534
|
+
host = downloader._host(url)
|
|
535
|
+
wait = host.cooldown_until - time.time()
|
|
536
|
+
if wait > 0:
|
|
537
|
+
self._send_code(429, {"Retry-After": str(math.ceil(wait))})
|
|
538
|
+
return
|
|
539
|
+
if not self.server.relay_slots.acquire(timeout=RELAY_WAIT):
|
|
540
|
+
self._send_code(503, {"Retry-After": "2"})
|
|
541
|
+
return
|
|
542
|
+
try:
|
|
543
|
+
if not host.slots.acquire(timeout=RELAY_WAIT):
|
|
544
|
+
self._send_code(503, {"Retry-After": "2"})
|
|
545
|
+
return
|
|
546
|
+
try:
|
|
547
|
+
host.pace()
|
|
548
|
+
self._relay_once(url, body, host)
|
|
549
|
+
finally:
|
|
550
|
+
host.slots.release()
|
|
551
|
+
finally:
|
|
552
|
+
self.server.relay_slots.release()
|
|
553
|
+
|
|
554
|
+
def _relay_once(self, url: str, body: bool, host: Any) -> None:
|
|
555
|
+
from .. import downloader
|
|
556
|
+
|
|
557
|
+
headers = {"User-Agent": downloader._USER_AGENT}
|
|
558
|
+
if self.headers.get("Range"):
|
|
559
|
+
headers["Range"] = self.headers["Range"]
|
|
560
|
+
req = urllib.request.Request(url, headers=headers, method="GET" if body else "HEAD")
|
|
561
|
+
started = False
|
|
562
|
+
try:
|
|
563
|
+
with _relay_opener.open(req, timeout=60) as resp:
|
|
564
|
+
self.send_response(resp.status)
|
|
565
|
+
started = True
|
|
566
|
+
self._common_headers(resp.headers.get("Content-Type", "application/octet-stream"))
|
|
567
|
+
for name in ("Content-Length", "Content-Range", "ETag", "Last-Modified"):
|
|
568
|
+
if resp.headers.get(name):
|
|
569
|
+
self.send_header(name, resp.headers[name])
|
|
570
|
+
if not resp.headers.get("Content-Length"):
|
|
571
|
+
self.close_connection = True
|
|
572
|
+
self.end_headers()
|
|
573
|
+
if body:
|
|
574
|
+
shutil.copyfileobj(resp, self.wfile, 65536)
|
|
575
|
+
except urllib.error.HTTPError as e:
|
|
576
|
+
if e.code in downloader.THROTTLE_CODES:
|
|
577
|
+
wait = downloader._retry_after(e) or 60.0
|
|
578
|
+
host.cool(wait)
|
|
579
|
+
if not started:
|
|
580
|
+
self._send_code(429, {"Retry-After": str(math.ceil(wait))})
|
|
581
|
+
elif not started:
|
|
582
|
+
self._send_code(e.code)
|
|
583
|
+
except OSError: # upstream unreachable / timed out, or the browser went away
|
|
584
|
+
self.close_connection = True
|
|
585
|
+
if not started:
|
|
586
|
+
self._send_code(502)
|
|
587
|
+
|
|
588
|
+
def _common_headers(self, ctype: str) -> None:
|
|
589
|
+
self.send_header("Content-Type", ctype)
|
|
590
|
+
self.send_header("Accept-Ranges", "bytes")
|
|
591
|
+
# Cross-origin isolation for DuckDB-WASM; `credentialless` (not require-corp) so
|
|
592
|
+
# browse images from hosts without CORP/CORS (JAXA DARTS) still load in <img>.
|
|
593
|
+
self.send_header("Cross-Origin-Opener-Policy", "same-origin")
|
|
594
|
+
self.send_header("Cross-Origin-Embedder-Policy", "credentialless")
|
|
595
|
+
|
|
596
|
+
def _send_bytes(self, path: Path, offset: int, length: int) -> None:
|
|
597
|
+
with open(path, "rb") as f:
|
|
598
|
+
f.seek(offset)
|
|
599
|
+
remaining = length
|
|
600
|
+
while remaining > 0:
|
|
601
|
+
buf = f.read(min(65536, remaining))
|
|
602
|
+
if not buf:
|
|
603
|
+
break
|
|
604
|
+
try:
|
|
605
|
+
self.wfile.write(buf)
|
|
606
|
+
except (BrokenPipeError, ConnectionResetError): # the browser aborted the read
|
|
607
|
+
self.close_connection = True
|
|
608
|
+
return
|
|
609
|
+
remaining -= len(buf)
|
|
610
|
+
|
|
611
|
+
def _send_code(self, code: int, headers: dict[str, str] | None = None) -> None:
|
|
612
|
+
self.send_response(code)
|
|
613
|
+
for k, v in (headers or {}).items():
|
|
614
|
+
self.send_header(k, v)
|
|
615
|
+
self.send_header("Content-Length", "0")
|
|
616
|
+
self.end_headers()
|
|
617
|
+
|
|
618
|
+
def log_message(self, fmt: str, *args: object) -> None: # noqa: ARG002
|
|
619
|
+
pass # suppress per-request noise; server start/stop is printed below
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
def _regions() -> dict[str, Any]:
|
|
623
|
+
from .. import reference as ref
|
|
624
|
+
|
|
625
|
+
return {
|
|
626
|
+
"regions": [
|
|
627
|
+
{
|
|
628
|
+
"id": r.id,
|
|
629
|
+
"name": r.name,
|
|
630
|
+
"pgda_site": r.pgda_site,
|
|
631
|
+
"zenodo_file": r.zenodo_file,
|
|
632
|
+
"zenodo_size": r.zenodo_size,
|
|
633
|
+
}
|
|
634
|
+
for r in ref.REGIONS
|
|
635
|
+
],
|
|
636
|
+
"pgda_kinds": list(ref.PGDA_KINDS),
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
def make_server(args: argparse.Namespace) -> tuple[LunarServer, ServerConfig]:
|
|
641
|
+
"""Bind a :class:`LunarServer` from the shared options (ValueError for a bad option)."""
|
|
642
|
+
import os
|
|
643
|
+
|
|
644
|
+
from ..config import RUNS_DIR_ENV
|
|
645
|
+
from ..constants import get_data_dir
|
|
646
|
+
|
|
647
|
+
explicit = Path(args.data_dir).expanduser() if args.data_dir else None
|
|
648
|
+
images = Path(args.images).expanduser() if args.images else None
|
|
649
|
+
if images is not None and not images.is_dir():
|
|
650
|
+
raise ValueError(f"--images: not a directory: {images}")
|
|
651
|
+
runs = args.runs_dir or os.environ.get(RUNS_DIR_ENV)
|
|
652
|
+
token = args.admin_token
|
|
653
|
+
if token == "auto":
|
|
654
|
+
token = secrets.token_urlsafe(18)
|
|
655
|
+
elif token is not None and len(token) < 12:
|
|
656
|
+
raise ValueError("--admin-token: use at least 12 characters (or 'auto')")
|
|
657
|
+
cfg = ServerConfig(
|
|
658
|
+
data_dir=explicit or get_data_dir(),
|
|
659
|
+
explicit_dir=explicit,
|
|
660
|
+
images_dir=images,
|
|
661
|
+
runs_dir=Path(runs).expanduser() if runs else None,
|
|
662
|
+
admin_token=token,
|
|
663
|
+
)
|
|
664
|
+
return LunarServer((args.host, args.port), cfg), cfg
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
def server_url(server: LunarServer, host: str) -> str:
|
|
668
|
+
"""URL for this machine's browser: wildcard binds map to localhost."""
|
|
669
|
+
port = server.server_address[1]
|
|
670
|
+
name = "localhost" if host in ("", "0.0.0.0", "::") else host
|
|
671
|
+
return f"http://{f'[{name}]' if ':' in name else name}:{port}"
|
|
672
|
+
|
|
673
|
+
|
|
674
|
+
def print_banner(server: LunarServer, cfg: ServerConfig, host: str, title: str) -> str:
|
|
675
|
+
url = server_url(server, host)
|
|
676
|
+
print(f"{title} -> {url}")
|
|
677
|
+
print(f" data dir : {cfg.data_dir}")
|
|
678
|
+
if cfg.images_dir is not None:
|
|
679
|
+
print(f" images : {cfg.images_dir}")
|
|
680
|
+
if cfg.runs_dir is not None:
|
|
681
|
+
print(f" runs dir : {cfg.runs_dir}")
|
|
682
|
+
if not (is_loopback(host) or host == "localhost"):
|
|
683
|
+
print(" network : remote clients are read-only (browse, search, view)")
|
|
684
|
+
if cfg.admin_token:
|
|
685
|
+
print(f" admin : {url}/?token={cfg.admin_token}")
|
|
686
|
+
return url
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
def run(args: argparse.Namespace) -> int:
|
|
690
|
+
try:
|
|
691
|
+
server, cfg = make_server(args)
|
|
692
|
+
except (ValueError, OSError) as exc:
|
|
693
|
+
print(exc)
|
|
694
|
+
return 1
|
|
695
|
+
url = print_banner(server, cfg, args.host, "Lunar Toolkit web app")
|
|
696
|
+
print(" Ctrl-C to stop.\n")
|
|
697
|
+
if not args.no_browser:
|
|
698
|
+
threading.Timer(0.4, webbrowser.open, args=[url]).start()
|
|
699
|
+
try:
|
|
700
|
+
server.serve_forever()
|
|
701
|
+
except KeyboardInterrupt:
|
|
702
|
+
print("\nStopped.")
|
|
703
|
+
finally:
|
|
704
|
+
server.jobs.close()
|
|
705
|
+
server.server_close()
|
|
706
|
+
return 0
|