lunar-toolkit 0.1.0.dev2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. lunar_toolkit/__init__.py +6 -0
  2. lunar_toolkit/asp.py +432 -0
  3. lunar_toolkit/cli/__init__.py +88 -0
  4. lunar_toolkit/cli/_jobs.py +437 -0
  5. lunar_toolkit/cli/_status.py +368 -0
  6. lunar_toolkit/cli/_utils.py +128 -0
  7. lunar_toolkit/cli/app.py +205 -0
  8. lunar_toolkit/cli/build_db.py +130 -0
  9. lunar_toolkit/cli/download.py +111 -0
  10. lunar_toolkit/cli/kaguya.py +253 -0
  11. lunar_toolkit/cli/ode_search.py +144 -0
  12. lunar_toolkit/cli/pairs.py +236 -0
  13. lunar_toolkit/cli/process.py +138 -0
  14. lunar_toolkit/cli/query.py +70 -0
  15. lunar_toolkit/cli/reference.py +233 -0
  16. lunar_toolkit/cli/search.py +90 -0
  17. lunar_toolkit/cli/serve.py +706 -0
  18. lunar_toolkit/cli/sfs.py +344 -0
  19. lunar_toolkit/cli/sfs_report.py +148 -0
  20. lunar_toolkit/cli/shadowcam.py +246 -0
  21. lunar_toolkit/cli/stereo.py +85 -0
  22. lunar_toolkit/cli/workspace.py +84 -0
  23. lunar_toolkit/config.py +345 -0
  24. lunar_toolkit/constants.py +217 -0
  25. lunar_toolkit/csm.py +340 -0
  26. lunar_toolkit/downloader.py +447 -0
  27. lunar_toolkit/geometry.py +556 -0
  28. lunar_toolkit/isis.py +222 -0
  29. lunar_toolkit/kaguya/__init__.py +45 -0
  30. lunar_toolkit/kaguya/mi.py +274 -0
  31. lunar_toolkit/kaguya/tc.py +370 -0
  32. lunar_toolkit/lroc/__init__.py +6 -0
  33. lunar_toolkit/lroc/cumindex.py +1197 -0
  34. lunar_toolkit/lroc/nac.py +702 -0
  35. lunar_toolkit/observation.py +127 -0
  36. lunar_toolkit/ode.py +485 -0
  37. lunar_toolkit/provenance.py +312 -0
  38. lunar_toolkit/reference.py +1111 -0
  39. lunar_toolkit/selection.py +714 -0
  40. lunar_toolkit/sfs.py +1534 -0
  41. lunar_toolkit/sfs_products.py +676 -0
  42. lunar_toolkit/sfs_report.py +1328 -0
  43. lunar_toolkit/shadowcam/__init__.py +29 -0
  44. lunar_toolkit/shadowcam/sc.py +392 -0
  45. lunar_toolkit/static/index.html +3259 -0
  46. lunar_toolkit/stereo.py +519 -0
  47. lunar_toolkit/stereo_dem.py +145 -0
  48. lunar_toolkit/tools.py +264 -0
  49. lunar_toolkit/workspace.py +1173 -0
  50. lunar_toolkit-0.1.0.dev2.dist-info/METADATA +177 -0
  51. lunar_toolkit-0.1.0.dev2.dist-info/RECORD +55 -0
  52. lunar_toolkit-0.1.0.dev2.dist-info/WHEEL +5 -0
  53. lunar_toolkit-0.1.0.dev2.dist-info/entry_points.txt +2 -0
  54. lunar_toolkit-0.1.0.dev2.dist-info/licenses/LICENSE +28 -0
  55. lunar_toolkit-0.1.0.dev2.dist-info/top_level.txt +1 -0
@@ -0,0 +1,706 @@
1
+ """serve - the web app, Range-capable Parquet / image serving, a PDS image relay, status and jobs."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import hmac
7
+ import http.server
8
+ import ipaddress
9
+ import json
10
+ import math
11
+ import secrets
12
+ import shutil
13
+ import socket
14
+ import threading
15
+ import time
16
+ import urllib.error
17
+ import urllib.parse
18
+ import urllib.request
19
+ import webbrowser
20
+ from dataclasses import dataclass
21
+ from http.client import HTTPMessage
22
+ from pathlib import Path
23
+ from typing import IO, Any
24
+
25
+ from ._jobs import TERMINAL, JobContext, JobManager, submit
26
+ from ._status import StatusCache, StatusDirs
27
+
28
+ COMMAND = "serve"
29
+
30
+ _STATIC_DIR = Path(__file__).parent.parent / "static"
31
+ _INDEX = _STATIC_DIR / "index.html"
32
+
33
+ _IMAGE_TYPES = {
34
+ ".tif": "image/tiff",
35
+ ".tiff": "image/tiff",
36
+ ".png": "image/png",
37
+ ".jpg": "image/jpeg",
38
+ ".jpeg": "image/jpeg",
39
+ }
40
+ # Image hosts the page may fetch through /remote: pds.mcp.nasa.gov drops its CORS header
41
+ # whenever the request carries extra headers (browsers always send `priority`).
42
+ _RELAY_HOSTS = (
43
+ "pds.mcp.nasa.gov",
44
+ "pds.lroc.im-ldi.com",
45
+ "pds.shadowcam.im-ldi.com",
46
+ "data.darts.isas.jaxa.jp",
47
+ )
48
+ _MAX_LISTED_IMAGES = 2000
49
+ RELAY_MAX = 6
50
+ RELAY_WAIT = 30.0
51
+ MAX_BODY = 64 * 1024
52
+ SSE_KEEPALIVE = 15.0
53
+ SSE_MIN_INTERVAL = 0.25
54
+ _LOCAL_NAMES = ("localhost", "127.0.0.1", "::1")
55
+
56
+
57
+ def add_server_args(p: argparse.ArgumentParser, host: str, port: int) -> None:
58
+ """Options shared by ``lunar serve`` and ``lunar app``."""
59
+ p.add_argument(
60
+ "--port",
61
+ type=int,
62
+ default=port,
63
+ metavar="PORT",
64
+ help=f"Port to listen on (default: {port or 'any free'}).",
65
+ )
66
+ p.add_argument(
67
+ "--host",
68
+ default=host,
69
+ metavar="HOST",
70
+ help=f"Address to bind (default: {host}); 0.0.0.0 serves the network (read-only there).",
71
+ )
72
+ p.add_argument(
73
+ "--data-dir",
74
+ default=None,
75
+ metavar="DIR",
76
+ help="Directory containing parquet files. Defaults to LUNAR_DATA_DIR or the user data dir.",
77
+ )
78
+ p.add_argument(
79
+ "--images",
80
+ default=None,
81
+ metavar="DIR",
82
+ help="Also serve the GeoTIFF / PNG / JPEG files under DIR (with Range) for the Image view.",
83
+ )
84
+ p.add_argument(
85
+ "--runs-dir",
86
+ default=None,
87
+ metavar="DIR",
88
+ help="SfS runs listed in the Settings tab (default: LUNAR_RUNS_DIR).",
89
+ )
90
+ p.add_argument(
91
+ "--admin-token",
92
+ default=None,
93
+ metavar="TOKEN",
94
+ help="Let remote clients with this token (Authorization: Bearer TOKEN or ?token=TOKEN) "
95
+ "start jobs and see local paths; 'auto' generates one.",
96
+ )
97
+
98
+
99
+ def add_parser(sub: argparse._SubParsersAction) -> None:
100
+ p = sub.add_parser(
101
+ COMMAND,
102
+ help="Serve the Lunar Toolkit web app with local parquet files.",
103
+ formatter_class=argparse.RawDescriptionHelpFormatter,
104
+ epilog="""
105
+ Examples
106
+ --------
107
+ lunar serve
108
+ lunar serve --port 9000
109
+ lunar serve --data-dir /mnt/lroc-data
110
+ lunar serve --images ~/sfs_runs # browse local GeoTIFF / COG results in the Image view
111
+ lunar serve --host 0.0.0.0 # share on the network; remote clients are read-only
112
+ lunar serve --host 0.0.0.0 --admin-token auto # ... and print a token for remote admin
113
+
114
+ Admin (the Settings tab: build indexes, download, see local paths) is allowed from this
115
+ machine (loopback) and for requests carrying --admin-token; everyone else can browse,
116
+ search and view. Use `lunar app` for a desktop window.
117
+ """,
118
+ )
119
+ add_server_args(p, "localhost", 8080)
120
+ p.add_argument("--no-browser", action="store_true", help="Don't open a browser tab automatically.")
121
+
122
+
123
+ def _byte_range(header: str, size: int) -> tuple[int, int] | tuple[()] | None:
124
+ """``(start, length)`` for a single ``bytes=`` Range, ``()`` without one, None if unsatisfiable."""
125
+ if not header.startswith("bytes="):
126
+ return ()
127
+ first, _, last = header[6:].split(",")[0].strip().partition("-")
128
+ try:
129
+ if not first: # suffix range: the last N bytes
130
+ n = int(last)
131
+ start, end = max(size - n, 0), size - 1
132
+ else:
133
+ start = int(first)
134
+ end = min(int(last), size - 1) if last else size - 1
135
+ except ValueError:
136
+ return None
137
+ if start >= size or end < start:
138
+ return None
139
+ return start, end - start + 1
140
+
141
+
142
+ def _image_path(root: Path, rel: str) -> Path | None:
143
+ """The image file ``rel`` (URL-quoted, relative) under ``root``, or None if it escapes or isn't one."""
144
+ parts = urllib.parse.unquote(rel).replace("\\", "/").split("/")
145
+ if not rel or any(not part or part.startswith(".") for part in parts):
146
+ return None
147
+ base = root.resolve()
148
+ path = base.joinpath(*parts).resolve()
149
+ if not path.is_relative_to(base) or path.suffix.lower() not in _IMAGE_TYPES or not path.is_file():
150
+ return None
151
+ return path
152
+
153
+
154
+ def _list_images(root: Path) -> list[str]:
155
+ """Relative POSIX paths of the image files under ``root`` (hidden entries skipped)."""
156
+ base = root.resolve()
157
+ out = []
158
+ for path in sorted(base.rglob("*")):
159
+ rel = path.relative_to(base)
160
+ if any(part.startswith(".") for part in rel.parts) or path.suffix.lower() not in _IMAGE_TYPES:
161
+ continue
162
+ if path.is_file() and path.resolve().is_relative_to(base):
163
+ out.append(rel.as_posix())
164
+ if len(out) >= _MAX_LISTED_IMAGES:
165
+ break
166
+ return out
167
+
168
+
169
+ def _report_path(runs_dir: Path, name: str) -> Path | None:
170
+ """``<runs_dir>/<name>/report/report.html`` if *name* is a plain run dir name and it exists."""
171
+ name = urllib.parse.unquote(name)
172
+ if not name or "/" in name or "\\" in name or name.startswith("."):
173
+ return None
174
+ base = runs_dir.resolve()
175
+ path = (base / name / "report" / "report.html").resolve()
176
+ return path if path.is_relative_to(base) and path.is_file() else None
177
+
178
+
179
+ def _relay_allowed(url: str) -> bool:
180
+ """Only https image files on the PDS / JAXA image hosts are relayed."""
181
+ try:
182
+ u = urllib.parse.urlsplit(url)
183
+ port = u.port
184
+ except ValueError:
185
+ return False
186
+ return (
187
+ u.scheme == "https"
188
+ and u.hostname in _RELAY_HOSTS
189
+ and port is None
190
+ and not u.username
191
+ and Path(u.path).suffix.lower() in _IMAGE_TYPES
192
+ )
193
+
194
+
195
+ class _RelayRedirect(urllib.request.HTTPRedirectHandler):
196
+ def redirect_request(
197
+ self,
198
+ req: urllib.request.Request,
199
+ fp: IO[bytes],
200
+ code: int,
201
+ msg: str,
202
+ headers: HTTPMessage,
203
+ newurl: str,
204
+ ) -> urllib.request.Request | None:
205
+ if not _relay_allowed(newurl):
206
+ raise urllib.error.HTTPError(newurl, 403, "redirect outside the relay hosts", headers, fp)
207
+ return super().redirect_request(req, fp, code, msg, headers, newurl)
208
+
209
+
210
+ _relay_opener = urllib.request.build_opener(_RelayRedirect)
211
+
212
+
213
+ def is_loopback(addr: str) -> bool:
214
+ """Whether client address *addr* is this machine (IPv4, IPv6 or IPv4-mapped IPv6)."""
215
+ try:
216
+ ip = ipaddress.ip_address(addr.split("%", 1)[0])
217
+ except ValueError:
218
+ return False
219
+ if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
220
+ ip = ip.ipv4_mapped
221
+ return ip.is_loopback
222
+
223
+
224
+ def _local_host_header(host: str) -> bool:
225
+ """A ``Host`` header naming this machine (guards loopback admin against DNS rebinding)."""
226
+ try:
227
+ name = urllib.parse.urlsplit("//" + host).hostname or ""
228
+ except ValueError:
229
+ return False
230
+ return name in _LOCAL_NAMES or name.endswith(".localhost")
231
+
232
+
233
+ @dataclass
234
+ class ServerConfig:
235
+ """What a :class:`LunarServer` serves and where its jobs write."""
236
+
237
+ data_dir: Path
238
+ explicit_dir: Path | None = None
239
+ images_dir: Path | None = None
240
+ runs_dir: Path | None = None
241
+ admin_token: str | None = None
242
+ cache_dir: Path | None = None
243
+ download_dir: Path | None = None
244
+ reference_dir: Path | None = None
245
+
246
+ def __post_init__(self) -> None:
247
+ from ..constants import DEFAULT_CACHE_DIR
248
+ from ..reference import REFERENCE_CACHE_DIR
249
+
250
+ self.cache_dir = self.cache_dir or DEFAULT_CACHE_DIR
251
+ self.download_dir = self.download_dir or self.cache_dir / "downloads"
252
+ self.reference_dir = self.reference_dir or REFERENCE_CACHE_DIR
253
+
254
+
255
+ class LunarServer(http.server.ThreadingHTTPServer):
256
+ """The HTTP server with its job manager, status cache and relay limit."""
257
+
258
+ daemon_threads = True
259
+ request_queue_size = 128
260
+
261
+ def __init__(self, address: tuple[str, int], cfg: ServerConfig, workers: int = 2) -> None:
262
+ self.cfg = cfg
263
+ assert cfg.cache_dir and cfg.download_dir and cfg.reference_dir
264
+ dirs = StatusDirs(cfg.data_dir, cfg.cache_dir, cfg.download_dir, cfg.reference_dir, cfg.runs_dir)
265
+ self.status = StatusCache(dirs)
266
+ self.jobs = JobManager(workers, on_finish=lambda job: self.status.invalidate())
267
+ self.job_context = JobContext(cfg.data_dir, cfg.download_dir, cfg.reference_dir)
268
+ self.relay_slots = threading.BoundedSemaphore(RELAY_MAX)
269
+ self.sdem_members: dict[int, list[dict[str, Any]]] = {}
270
+ if ":" in address[0]:
271
+ self.address_family = socket.AF_INET6
272
+ super().__init__(address, LunarHandler)
273
+
274
+ def close(self) -> None:
275
+ """Cancel jobs, stop serving and close the socket."""
276
+ self.jobs.close()
277
+ self.shutdown()
278
+ self.server_close()
279
+
280
+
281
+ class LunarHandler(http.server.BaseHTTPRequestHandler):
282
+ # Keep-alive: DuckDB-WASM and the image viewer issue thousands of small Range requests
283
+ protocol_version = "HTTP/1.1"
284
+ server: LunarServer
285
+
286
+ def client_ip(self) -> str:
287
+ return self.client_address[0]
288
+
289
+ def _query(self) -> dict[str, list[str]]:
290
+ return urllib.parse.parse_qs(self.path.partition("?")[2])
291
+
292
+ def _token(self) -> str | None:
293
+ auth = self.headers.get("Authorization", "")
294
+ if auth.lower().startswith("bearer "):
295
+ return auth[7:].strip()
296
+ return self._query().get("token", [None])[0]
297
+
298
+ def is_admin(self) -> bool:
299
+ """Loopback clients (with a local ``Host``) and holders of the admin token."""
300
+ token, given = self.server.cfg.admin_token, self._token()
301
+ if token and given and hmac.compare_digest(given.encode(), token.encode()):
302
+ return True
303
+ return is_loopback(self.client_ip()) and _local_host_header(self.headers.get("Host", ""))
304
+
305
+ def do_GET(self) -> None:
306
+ self._route(body=True)
307
+
308
+ def do_HEAD(self) -> None:
309
+ self._route(body=False)
310
+
311
+ def do_POST(self) -> None:
312
+ path = self.path.partition("?")[0]
313
+ if path != "/api/jobs":
314
+ self._error(404, "not found")
315
+ return
316
+ if not self._check_admin(write=True):
317
+ return
318
+ if "application/json" not in self.headers.get("Content-Type", ""):
319
+ self._error(415, "expected application/json")
320
+ return
321
+ try:
322
+ n = int(self.headers.get("Content-Length", "0"))
323
+ except ValueError:
324
+ n = -1
325
+ if not 0 < n <= MAX_BODY:
326
+ self._error(400 if n <= 0 else 413, "bad request body")
327
+ return
328
+ try:
329
+ payload = json.loads(self.rfile.read(n))
330
+ job = submit(self.server.jobs, payload, self.server.job_context)
331
+ except ValueError as exc: # also json.JSONDecodeError
332
+ self._error(409 if "already" in str(exc) else 400, str(exc))
333
+ return
334
+ except RuntimeError as exc:
335
+ self._error(503, str(exc))
336
+ return
337
+ self._json(job.to_dict(), code=201)
338
+
339
+ def do_DELETE(self) -> None:
340
+ path = self.path.partition("?")[0]
341
+ parts = path.strip("/").split("/")
342
+ if len(parts) != 3 or parts[:2] != ["api", "jobs"]:
343
+ self._error(404, "not found")
344
+ return
345
+ if not self._check_admin(write=True):
346
+ return
347
+ job = self.server.jobs.cancel(parts[2])
348
+ if job is None:
349
+ self._error(404, "no such job")
350
+ return
351
+ self._json(job.to_dict())
352
+
353
+ def do_OPTIONS(self) -> None:
354
+ self._error(405, "method not allowed")
355
+
356
+ def _check_admin(self, write: bool = False) -> bool:
357
+ if not self.is_admin():
358
+ self._error(403, "read-only client: admin needs localhost or the --admin-token")
359
+ return False
360
+ origin = self.headers.get("Origin")
361
+ if write and origin and urllib.parse.urlsplit(origin).netloc != self.headers.get("Host", ""):
362
+ self._error(403, "cross-origin request")
363
+ return False
364
+ return True
365
+
366
+ def _route(self, body: bool) -> None:
367
+ path, _, query = self.path.partition("?")
368
+ cfg = self.server.cfg
369
+ if path in ("/", "/index.html"):
370
+ self._serve(_INDEX, "text/html; charset=utf-8", body)
371
+ elif path == "/api/config":
372
+ from ..constants import APP_VERSION
373
+
374
+ self._json(
375
+ {
376
+ "relay": list(_RELAY_HOSTS),
377
+ "images": cfg.images_dir is not None,
378
+ "admin": self.is_admin(),
379
+ "jobs": True,
380
+ "runs": cfg.runs_dir is not None,
381
+ "version": APP_VERSION,
382
+ },
383
+ body,
384
+ )
385
+ elif path == "/api/images":
386
+ self._json(_list_images(cfg.images_dir) if cfg.images_dir else [], body)
387
+ elif path.startswith("/images/") and cfg.images_dir is not None:
388
+ img = _image_path(cfg.images_dir, path[len("/images/") :])
389
+ if img is None:
390
+ self._send_code(404)
391
+ return
392
+ self._serve(img, _IMAGE_TYPES[img.suffix.lower()], body)
393
+ elif path == "/remote":
394
+ self._relay(urllib.parse.parse_qs(query).get("url", [""])[0], body)
395
+ elif path.startswith("/data/"):
396
+ from ..constants import find_parquet
397
+
398
+ name = path[len("/data/") :]
399
+ if not name or "/" in name or name.startswith("."):
400
+ self._send_code(404)
401
+ return
402
+ self._serve(find_parquet(name, cfg.data_dir), "application/octet-stream", body)
403
+ elif path == "/api/status":
404
+ refresh = self._query().get("refresh", ["0"])[0] == "1"
405
+ self._json(self.server.status.get(self.is_admin(), refresh=refresh), body)
406
+ elif path == "/api/regions":
407
+ self._json(_regions(), body)
408
+ elif path == "/api/sdem/members":
409
+ if self._check_admin():
410
+ self._sdem_members(self._query().get("region", [""])[0], body)
411
+ elif path.startswith("/api/jobs"):
412
+ self._jobs_get(path, body)
413
+ elif path.startswith("/runs/") and path.endswith("/report.html") and cfg.runs_dir is not None:
414
+ if not self._check_admin():
415
+ return
416
+ report = _report_path(cfg.runs_dir, path[len("/runs/") : -len("/report.html")])
417
+ if report is None:
418
+ self._send_code(404)
419
+ return
420
+ self._serve(report, "text/html; charset=utf-8", body)
421
+ else:
422
+ self._send_code(404)
423
+
424
+ def _jobs_get(self, path: str, body: bool) -> None:
425
+ admin = self.is_admin()
426
+ parts = path.strip("/").split("/")[2:]
427
+ jobs = self.server.jobs
428
+ if not parts:
429
+ self._json([j.to_dict(admin) for j in jobs.jobs()], body)
430
+ elif parts == ["events"]:
431
+ self._events(None, admin)
432
+ elif len(parts) in (1, 2) and parts[1:] in ([], ["events"]):
433
+ job = jobs.get(parts[0])
434
+ if job is None:
435
+ self._error(404, "no such job")
436
+ elif len(parts) == 2:
437
+ self._events(parts[0], admin)
438
+ else:
439
+ self._json(job.to_dict(admin), body)
440
+ else:
441
+ self._error(404, "not found")
442
+
443
+ def _events(self, jid: str | None, admin: bool) -> None:
444
+ """Server-Sent Events: ``event: jobs`` (all jobs) or ``event: job`` (one) on every change."""
445
+ jobs = self.server.jobs
446
+ self.send_response(200)
447
+ self._common_headers("text/event-stream")
448
+ self.send_header("Cache-Control", "no-cache")
449
+ self.end_headers()
450
+ self.close_connection = True
451
+ version = -1
452
+ data: Any
453
+ try:
454
+ while not jobs.closed:
455
+ new = jobs.wait(version, SSE_KEEPALIVE)
456
+ if new == version:
457
+ self.wfile.write(b": keepalive\n\n")
458
+ self.wfile.flush()
459
+ continue
460
+ version = new
461
+ if jid is None:
462
+ event, data = "jobs", [j.to_dict(admin) for j in jobs.jobs()]
463
+ final = False
464
+ else:
465
+ job = jobs.get(jid)
466
+ if job is None:
467
+ break
468
+ event, data, final = "job", job.to_dict(admin), job.status in TERMINAL
469
+ self.wfile.write(f"event: {event}\ndata: {json.dumps(data)}\n\n".encode())
470
+ self.wfile.flush()
471
+ if final:
472
+ break
473
+ time.sleep(SSE_MIN_INTERVAL)
474
+ except (BrokenPipeError, ConnectionResetError, OSError):
475
+ pass
476
+
477
+ def _sdem_members(self, region: str, body: bool) -> None:
478
+ from .. import reference as ref
479
+
480
+ try:
481
+ reg = ref.get_region(region)
482
+ except KeyError as exc:
483
+ self._error(400, str(exc.args[0] if exc.args else exc))
484
+ return
485
+ cache = self.server.sdem_members
486
+ if reg.id not in cache:
487
+ try:
488
+ df = ref.sdem_members(reg)
489
+ except OSError as exc:
490
+ self._error(502, f"could not read the zip directory: {exc}")
491
+ return
492
+ cache[reg.id] = [{"name": str(r["name"]), "size": int(r["size"])} for r in df.to_dict("records")]
493
+ self._json({"region": reg.id, "zip": reg.zenodo_file, "members": cache[reg.id]}, body)
494
+
495
+ def _serve(self, path: Path, ctype: str, body: bool = True) -> None:
496
+ if not path.exists():
497
+ self._send_code(404)
498
+ return
499
+ size = path.stat().st_size
500
+ span = _byte_range(self.headers.get("Range", ""), size)
501
+ if span is None:
502
+ self._send_code(416)
503
+ return
504
+ start, length = span if span else (0, size)
505
+ self.send_response(206 if span else 200)
506
+ self._common_headers(ctype)
507
+ self.send_header("Content-Length", str(length))
508
+ if span:
509
+ self.send_header("Content-Range", f"bytes {start}-{start + length - 1}/{size}")
510
+ self.end_headers()
511
+ if body:
512
+ self._send_bytes(path, start, length)
513
+
514
+ def _json(self, obj: object, body: bool = True, code: int = 200) -> None:
515
+ data = json.dumps(obj).encode()
516
+ self.send_response(code)
517
+ self._common_headers("application/json")
518
+ self.send_header("Cache-Control", "no-store")
519
+ self.send_header("Content-Length", str(len(data)))
520
+ self.end_headers()
521
+ if body:
522
+ self.wfile.write(data)
523
+
524
+ def _error(self, code: int, message: str) -> None:
525
+ self._json({"error": message}, self.command != "HEAD", code)
526
+
527
+ def _relay(self, url: str, body: bool) -> None:
528
+ """Forward one Range request, through the downloader's per-host slots, pacing and cooldown."""
529
+ from .. import downloader
530
+
531
+ if not _relay_allowed(url):
532
+ self._send_code(403)
533
+ return
534
+ host = downloader._host(url)
535
+ wait = host.cooldown_until - time.time()
536
+ if wait > 0:
537
+ self._send_code(429, {"Retry-After": str(math.ceil(wait))})
538
+ return
539
+ if not self.server.relay_slots.acquire(timeout=RELAY_WAIT):
540
+ self._send_code(503, {"Retry-After": "2"})
541
+ return
542
+ try:
543
+ if not host.slots.acquire(timeout=RELAY_WAIT):
544
+ self._send_code(503, {"Retry-After": "2"})
545
+ return
546
+ try:
547
+ host.pace()
548
+ self._relay_once(url, body, host)
549
+ finally:
550
+ host.slots.release()
551
+ finally:
552
+ self.server.relay_slots.release()
553
+
554
+ def _relay_once(self, url: str, body: bool, host: Any) -> None:
555
+ from .. import downloader
556
+
557
+ headers = {"User-Agent": downloader._USER_AGENT}
558
+ if self.headers.get("Range"):
559
+ headers["Range"] = self.headers["Range"]
560
+ req = urllib.request.Request(url, headers=headers, method="GET" if body else "HEAD")
561
+ started = False
562
+ try:
563
+ with _relay_opener.open(req, timeout=60) as resp:
564
+ self.send_response(resp.status)
565
+ started = True
566
+ self._common_headers(resp.headers.get("Content-Type", "application/octet-stream"))
567
+ for name in ("Content-Length", "Content-Range", "ETag", "Last-Modified"):
568
+ if resp.headers.get(name):
569
+ self.send_header(name, resp.headers[name])
570
+ if not resp.headers.get("Content-Length"):
571
+ self.close_connection = True
572
+ self.end_headers()
573
+ if body:
574
+ shutil.copyfileobj(resp, self.wfile, 65536)
575
+ except urllib.error.HTTPError as e:
576
+ if e.code in downloader.THROTTLE_CODES:
577
+ wait = downloader._retry_after(e) or 60.0
578
+ host.cool(wait)
579
+ if not started:
580
+ self._send_code(429, {"Retry-After": str(math.ceil(wait))})
581
+ elif not started:
582
+ self._send_code(e.code)
583
+ except OSError: # upstream unreachable / timed out, or the browser went away
584
+ self.close_connection = True
585
+ if not started:
586
+ self._send_code(502)
587
+
588
+ def _common_headers(self, ctype: str) -> None:
589
+ self.send_header("Content-Type", ctype)
590
+ self.send_header("Accept-Ranges", "bytes")
591
+ # Cross-origin isolation for DuckDB-WASM; `credentialless` (not require-corp) so
592
+ # browse images from hosts without CORP/CORS (JAXA DARTS) still load in <img>.
593
+ self.send_header("Cross-Origin-Opener-Policy", "same-origin")
594
+ self.send_header("Cross-Origin-Embedder-Policy", "credentialless")
595
+
596
+ def _send_bytes(self, path: Path, offset: int, length: int) -> None:
597
+ with open(path, "rb") as f:
598
+ f.seek(offset)
599
+ remaining = length
600
+ while remaining > 0:
601
+ buf = f.read(min(65536, remaining))
602
+ if not buf:
603
+ break
604
+ try:
605
+ self.wfile.write(buf)
606
+ except (BrokenPipeError, ConnectionResetError): # the browser aborted the read
607
+ self.close_connection = True
608
+ return
609
+ remaining -= len(buf)
610
+
611
+ def _send_code(self, code: int, headers: dict[str, str] | None = None) -> None:
612
+ self.send_response(code)
613
+ for k, v in (headers or {}).items():
614
+ self.send_header(k, v)
615
+ self.send_header("Content-Length", "0")
616
+ self.end_headers()
617
+
618
+ def log_message(self, fmt: str, *args: object) -> None: # noqa: ARG002
619
+ pass # suppress per-request noise; server start/stop is printed below
620
+
621
+
622
+ def _regions() -> dict[str, Any]:
623
+ from .. import reference as ref
624
+
625
+ return {
626
+ "regions": [
627
+ {
628
+ "id": r.id,
629
+ "name": r.name,
630
+ "pgda_site": r.pgda_site,
631
+ "zenodo_file": r.zenodo_file,
632
+ "zenodo_size": r.zenodo_size,
633
+ }
634
+ for r in ref.REGIONS
635
+ ],
636
+ "pgda_kinds": list(ref.PGDA_KINDS),
637
+ }
638
+
639
+
640
+ def make_server(args: argparse.Namespace) -> tuple[LunarServer, ServerConfig]:
641
+ """Bind a :class:`LunarServer` from the shared options (ValueError for a bad option)."""
642
+ import os
643
+
644
+ from ..config import RUNS_DIR_ENV
645
+ from ..constants import get_data_dir
646
+
647
+ explicit = Path(args.data_dir).expanduser() if args.data_dir else None
648
+ images = Path(args.images).expanduser() if args.images else None
649
+ if images is not None and not images.is_dir():
650
+ raise ValueError(f"--images: not a directory: {images}")
651
+ runs = args.runs_dir or os.environ.get(RUNS_DIR_ENV)
652
+ token = args.admin_token
653
+ if token == "auto":
654
+ token = secrets.token_urlsafe(18)
655
+ elif token is not None and len(token) < 12:
656
+ raise ValueError("--admin-token: use at least 12 characters (or 'auto')")
657
+ cfg = ServerConfig(
658
+ data_dir=explicit or get_data_dir(),
659
+ explicit_dir=explicit,
660
+ images_dir=images,
661
+ runs_dir=Path(runs).expanduser() if runs else None,
662
+ admin_token=token,
663
+ )
664
+ return LunarServer((args.host, args.port), cfg), cfg
665
+
666
+
667
+ def server_url(server: LunarServer, host: str) -> str:
668
+ """URL for this machine's browser: wildcard binds map to localhost."""
669
+ port = server.server_address[1]
670
+ name = "localhost" if host in ("", "0.0.0.0", "::") else host
671
+ return f"http://{f'[{name}]' if ':' in name else name}:{port}"
672
+
673
+
674
+ def print_banner(server: LunarServer, cfg: ServerConfig, host: str, title: str) -> str:
675
+ url = server_url(server, host)
676
+ print(f"{title} -> {url}")
677
+ print(f" data dir : {cfg.data_dir}")
678
+ if cfg.images_dir is not None:
679
+ print(f" images : {cfg.images_dir}")
680
+ if cfg.runs_dir is not None:
681
+ print(f" runs dir : {cfg.runs_dir}")
682
+ if not (is_loopback(host) or host == "localhost"):
683
+ print(" network : remote clients are read-only (browse, search, view)")
684
+ if cfg.admin_token:
685
+ print(f" admin : {url}/?token={cfg.admin_token}")
686
+ return url
687
+
688
+
689
+ def run(args: argparse.Namespace) -> int:
690
+ try:
691
+ server, cfg = make_server(args)
692
+ except (ValueError, OSError) as exc:
693
+ print(exc)
694
+ return 1
695
+ url = print_banner(server, cfg, args.host, "Lunar Toolkit web app")
696
+ print(" Ctrl-C to stop.\n")
697
+ if not args.no_browser:
698
+ threading.Timer(0.4, webbrowser.open, args=[url]).start()
699
+ try:
700
+ server.serve_forever()
701
+ except KeyboardInterrupt:
702
+ print("\nStopped.")
703
+ finally:
704
+ server.jobs.close()
705
+ server.server_close()
706
+ return 0