iowap-node 2.3.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- iowap_node-2.3.20.dist-info/METADATA +126 -0
- iowap_node-2.3.20.dist-info/RECORD +28 -0
- iowap_node-2.3.20.dist-info/WHEEL +4 -0
- iowap_node-2.3.20.dist-info/entry_points.txt +3 -0
- iowap_node-2.3.20.dist-info/licenses/LICENSE +21 -0
- nodes/__init__.py +1 -0
- nodes/common/__init__.py +0 -0
- nodes/common/capability.py +520 -0
- nodes/common/cli/__init__.py +0 -0
- nodes/common/cli/cli_artifact.py +42 -0
- nodes/common/cli/cli_bridge.py +264 -0
- nodes/common/cli/cli_capabilities.py +229 -0
- nodes/common/cli/cli_docs.py +122 -0
- nodes/common/cli/cli_file.py +434 -0
- nodes/common/cli/cli_hp.py +546 -0
- nodes/common/cli/cli_node.py +249 -0
- nodes/common/cli/cli_route.py +92 -0
- nodes/common/cli/cli_server.py +248 -0
- nodes/common/cli/cli_task.py +191 -0
- nodes/common/cli/cli_update.py +48 -0
- nodes/common/envelope.py +168 -0
- nodes/common/file_serve.py +648 -0
- nodes/common/handler_runner.py +497 -0
- nodes/common/node_cli.py +1311 -0
- nodes/common/node_config.py +856 -0
- nodes/common/node_daemon.py +946 -0
- nodes/common/node_utils.py +448 -0
- nodes/common/relay_client.py +1246 -0
|
@@ -0,0 +1,648 @@
|
|
|
1
|
+
"""T-166: Ephemeral file serve — CLI→daemon handoff for ``hp put`` bridge.
|
|
2
|
+
|
|
3
|
+
The sending node stages the file to ``~/.relay/serve/{transfer_id}``
|
|
4
|
+
and registers a temp route whose upstream points at the serve server
|
|
5
|
+
inside the DAEMON (F1 — the CLI process dies after the envelope, so
|
|
6
|
+
the endpoint must live in the long-running service). The receiver
|
|
7
|
+
pulls through the relay proxy (F7). After ``max_downloads`` successful
|
|
8
|
+
downloads (F6, default 1) the daemon deletes file + manifest and
|
|
9
|
+
unregisters the route (the TTL sweep T-125 is the backstop).
|
|
10
|
+
|
|
11
|
+
FROZEN (Phase 1, T-166) — every signature in this module is immutable
|
|
12
|
+
for Phase 2 (scaffold) and Phase 3 (integrate):
|
|
13
|
+
|
|
14
|
+
* ``serve_port()`` — env ``IOWAP_SERVE_PORT``, else 8792.
|
|
15
|
+
* ``read_manifest()`` — read serve.json; ``None`` when missing/
|
|
16
|
+
invalid (the normal case without a daemon).
|
|
17
|
+
* ``write_manifest(port)`` — write atomically (tmp + os.replace).
|
|
18
|
+
* ``new_transfer_id()`` — ``secrets.token_urlsafe(16)`` → 22 chars
|
|
19
|
+
(≤64-char limit of the route registry).
|
|
20
|
+
* ``stage_file(src)`` — stage to ``SERVE_DIR``, compute sha256
|
|
21
|
+
(F8), write the F6 manifest
|
|
22
|
+
``{file, size, sha256, max_downloads}``.
|
|
23
|
+
* ``build_storage_ref(...)`` — F4 shape ``{type, node_id, path, expires_at}``
|
|
24
|
+
(envelope v1 unchanged).
|
|
25
|
+
* ``proxy_url(...)`` — F7 URL shape, identical to the
|
|
26
|
+
``unregister_temp_route`` path (T-126).
|
|
27
|
+
* ``probe_serve(port)`` — ``GET /health``; ``True`` iff HTTP 200.
|
|
28
|
+
* ``EphemeralServeHandler`` — skeleton; Phase 3 (plan task 2) fills
|
|
29
|
+
POST ``/transfer/{id}`` + GET ``/health``.
|
|
30
|
+
* ``start_serve_thread()`` — skeleton; Phase 3 (plan task 3) fills the
|
|
31
|
+
bind + manifest write + daemon thread.
|
|
32
|
+
* ``unregister_after_transfer`` — best-effort route deletion (F6),
|
|
33
|
+
log errors only (TTL is the backstop).
|
|
34
|
+
|
|
35
|
+
Phase-1 refinements (FROZEN, documented in the plan under "Phase 1"):
|
|
36
|
+
|
|
37
|
+
* ``stage_file`` gets the keyword-only argument ``max_downloads=1``
|
|
38
|
+
(F6) — call-compatible with the plan snippet ``stage_file(path)``.
|
|
39
|
+
* ``read_manifest()`` validates the port strictly (int 1–65535, bool
|
|
40
|
+
explicitly excluded) — cmd_put (task 4) gets a typed guarantee.
|
|
41
|
+
* ``serve_port()`` tolerates invalid env values with WARNING + default
|
|
42
|
+
(house pattern: relay_client._effective_config) instead of SystemExit.
|
|
43
|
+
* ``probe_serve`` counts only HTTP 200 as "reachable".
|
|
44
|
+
"""
|
|
45
|
+
from __future__ import annotations
|
|
46
|
+
|
|
47
|
+
import errno
|
|
48
|
+
import hashlib
|
|
49
|
+
import json
|
|
50
|
+
import logging
|
|
51
|
+
import os
|
|
52
|
+
import re
|
|
53
|
+
import secrets
|
|
54
|
+
import shutil
|
|
55
|
+
import threading
|
|
56
|
+
import time
|
|
57
|
+
from collections.abc import Callable
|
|
58
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
59
|
+
from pathlib import Path
|
|
60
|
+
|
|
61
|
+
import httpx
|
|
62
|
+
|
|
63
|
+
from nodes.common.relay_client import RelayClient
|
|
64
|
+
|
|
65
|
+
log = logging.getLogger("file-serve")
|
|
66
|
+
|
|
67
|
+
SERVE_PORT_DEFAULT = 8792
|
|
68
|
+
SERVE_HOST = "127.0.0.1" # Default — the relay proxy is the only client.
|
|
69
|
+
# T-166 live-smoke deviation (D8): the relay and the node run on different
|
|
70
|
+
# machines (LXC 903 vs. Hermes host). The proxy dials the registered
|
|
71
|
+
# upstream address VERBATIM — ``127.0.0.1`` points into the void there (502).
|
|
72
|
+
# ``IOWAP_SERVE_HOST`` overrides the bind AND advertise address (e.g. the
|
|
73
|
+
# LAN IP); the default remains the plan's single-host behavior.
|
|
74
|
+
|
|
75
|
+
# T-166 D9: source-IP allowlist (T-128 pattern from the storage node). When
|
|
76
|
+
# the serve binds to 0.0.0.0, the transfer endpoints are reachable in the
|
|
77
|
+
# LAN — only connections from the relay server IP (T-128: "the relay proxy
|
|
78
|
+
# is the only legitimate dialer") plus loopback for the local CLI probe
|
|
79
|
+
# path are accepted. Fail closed when the IP is not resolvable.
|
|
80
|
+
# X-Forwarded-For is never evaluated — the socket peer is
|
|
81
|
+
# authoritative (otherwise spoofable).
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def serve_host() -> str:
|
|
85
|
+
"""Bind/advertise host: env ``IOWAP_SERVE_HOST``, else ``SERVE_HOST``.
|
|
86
|
+
|
|
87
|
+
House pattern like ``serve_port()``: a nonsensical env value (empty,
|
|
88
|
+
whitespace, >255 chars) → WARNING + default, no hard crash.
|
|
89
|
+
Evaluated in the daemon AND in the CLI — both run on the same
|
|
90
|
+
machine, so the same env set yields the same address.
|
|
91
|
+
"""
|
|
92
|
+
raw = os.environ.get("IOWAP_SERVE_HOST")
|
|
93
|
+
if raw is None:
|
|
94
|
+
return SERVE_HOST
|
|
95
|
+
host = raw.strip()
|
|
96
|
+
if not host or len(host) > 255:
|
|
97
|
+
log.warning("ignoring invalid IOWAP_SERVE_HOST=%r", raw)
|
|
98
|
+
return SERVE_HOST
|
|
99
|
+
return host
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _relay_host_ip() -> str | None:
|
|
103
|
+
"""First IPv4 of the relay hostname from the node relay config (D9).
|
|
104
|
+
|
|
105
|
+
Order like ``_effective_config()``/storage-node T-128: env
|
|
106
|
+
``RELAY_BASE_URL`` → relay_config.json ``base_url`` → mDNS discovery.
|
|
107
|
+
``None`` when none of these is resolvable (the caller then decides
|
|
108
|
+
to fail closed).
|
|
109
|
+
"""
|
|
110
|
+
import socket
|
|
111
|
+
from urllib.parse import urlparse
|
|
112
|
+
|
|
113
|
+
from nodes.common.relay_client import (
|
|
114
|
+
_discover_relay_mdns,
|
|
115
|
+
_effective_config,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
cfg = _effective_config()
|
|
119
|
+
relay_url = str(cfg.get("base_url") or "")
|
|
120
|
+
if not relay_url:
|
|
121
|
+
discovered = _discover_relay_mdns()
|
|
122
|
+
if discovered:
|
|
123
|
+
relay_url = discovered
|
|
124
|
+
if not relay_url:
|
|
125
|
+
return None
|
|
126
|
+
host = urlparse(relay_url).hostname
|
|
127
|
+
if not host:
|
|
128
|
+
return None
|
|
129
|
+
try:
|
|
130
|
+
infos = socket.getaddrinfo(
|
|
131
|
+
host, None, family=socket.AF_INET, type=socket.SOCK_STREAM
|
|
132
|
+
)
|
|
133
|
+
except socket.gaierror:
|
|
134
|
+
return None
|
|
135
|
+
for info in infos:
|
|
136
|
+
if info[0] == socket.AF_INET:
|
|
137
|
+
return str(info[4][0])
|
|
138
|
+
return None
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def serve_allow_ip() -> str | None:
|
|
142
|
+
"""The single allowed client IP at the serve endpoint (D9).
|
|
143
|
+
|
|
144
|
+
Chain: env ``IOWAP_SERVE_ALLOW`` (explicit override) → relay IP from
|
|
145
|
+
the node config. ``None`` = not resolvable → LAN peers are rejected
|
|
146
|
+
(fail closed for the remote path); the local probe path
|
|
147
|
+
(loopback/advertise IP) stays servable regardless (D8 lesson:
|
|
148
|
+
``hp put`` probes the advertise address — the peer is the node's
|
|
149
|
+
own LAN IP). Cached at server bind time (T-128 semantics: relay IP
|
|
150
|
+
change → daemon restart).
|
|
151
|
+
"""
|
|
152
|
+
raw = os.environ.get("IOWAP_SERVE_ALLOW")
|
|
153
|
+
if raw and raw.strip():
|
|
154
|
+
ip = raw.strip().split(",")[0].strip()
|
|
155
|
+
try:
|
|
156
|
+
import ipaddress
|
|
157
|
+
|
|
158
|
+
ipaddress.IPv4Address(ip)
|
|
159
|
+
except ValueError:
|
|
160
|
+
log.warning("ignoring invalid IOWAP_SERVE_ALLOW=%r", raw)
|
|
161
|
+
else:
|
|
162
|
+
return ip
|
|
163
|
+
return _relay_host_ip()
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _allowed_peers() -> frozenset[str]:
|
|
167
|
+
"""Complete peer allowlist (D9): relay + advertise + loopback.
|
|
168
|
+
|
|
169
|
+
Legitimate clients at the serve endpoint:
|
|
170
|
+
|
|
171
|
+
* the relay proxy (relay IP — ``serve_allow_ip()``),
|
|
172
|
+
* the node CLI on the same host (loopback; disguised via the
|
|
173
|
+
advertise address as the node's own LAN IP — D8: ``probe_serve``
|
|
174
|
+
targets the advertise address, so the node's own advertise IP is
|
|
175
|
+
allowed too),
|
|
176
|
+
* loopback in general (probe, local tools).
|
|
177
|
+
|
|
178
|
+
A relay-IP outage degrades only the remote path (WARNING at bind),
|
|
179
|
+
never the local one.
|
|
180
|
+
"""
|
|
181
|
+
import socket
|
|
182
|
+
|
|
183
|
+
peers = {"127.0.0.1", "::1"}
|
|
184
|
+
relay_ip = serve_allow_ip()
|
|
185
|
+
if relay_ip:
|
|
186
|
+
peers.add(relay_ip)
|
|
187
|
+
host = serve_host()
|
|
188
|
+
if host not in ("", "0.0.0.0", "127.0.0.1", "localhost"):
|
|
189
|
+
try:
|
|
190
|
+
peers.add(socket.gethostbyname(host))
|
|
191
|
+
except OSError:
|
|
192
|
+
log.debug("cannot resolve advertise host %r — skipping in allowlist", host)
|
|
193
|
+
return frozenset(peers)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
SERVE_DIR = Path.home() / ".relay" / "serve"
|
|
197
|
+
MANIFEST_PATH = Path.home() / ".relay" / "serve.json"
|
|
198
|
+
|
|
199
|
+
_ROUTE_BASE = "/relay/v2/dashboard/api/node-routes"
|
|
200
|
+
|
|
201
|
+
# --- Runtime state (task 2/3) -------------------------------------------------
|
|
202
|
+
# TTL for staged leftovers without a route (sweep backstop, F6); sweep
|
|
203
|
+
# interval inside the serve thread. transfer_id: exactly 22 url-safe chars (F2).
|
|
204
|
+
_SERVE_TTL_SECONDS = 3600.0
|
|
205
|
+
_SWEEP_INTERVAL = 60.0
|
|
206
|
+
_TRANSFER_RE = re.compile(r"^/transfer/([A-Za-z0-9_-]{22})$")
|
|
207
|
+
|
|
208
|
+
# Daemon runtime: tests patch these module attributes (call-time lookup).
|
|
209
|
+
_serve_thread: threading.Thread | None = None
|
|
210
|
+
_serve_server: ThreadingHTTPServer | None = None
|
|
211
|
+
_ON_EXHAUSTED: Callable[[str], None] | None = None
|
|
212
|
+
_last_sweep = 0.0
|
|
213
|
+
_transfer_lock = threading.Lock()
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def set_on_exhausted(hook: Callable[[str], None] | None) -> None:
|
|
217
|
+
"""Hook ``(route_path) -> None`` after an exhausted transfer (F6).
|
|
218
|
+
|
|
219
|
+
The daemon wires ``unregister_after_transfer`` here (a closure over
|
|
220
|
+
its RelayClient); tests set a spy. Invocation happens in the request
|
|
221
|
+
thread AFTER the fully sent response — hook errors are logged and
|
|
222
|
+
never break the download.
|
|
223
|
+
"""
|
|
224
|
+
global _ON_EXHAUSTED
|
|
225
|
+
_ON_EXHAUSTED = hook
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def discard_staged(transfer_id: str) -> None:
|
|
229
|
+
"""Remove the staged file + transfer manifest (rollback after a register failure).
|
|
230
|
+
|
|
231
|
+
``hp put`` (task 4) calls this when the route registration fails —
|
|
232
|
+
no stale leftover without a route in SERVE_DIR (the sweep is the
|
|
233
|
+
backstop).
|
|
234
|
+
"""
|
|
235
|
+
(SERVE_DIR / transfer_id).unlink(missing_ok=True)
|
|
236
|
+
(SERVE_DIR / (transfer_id + ".json")).unlink(missing_ok=True)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _sweep_stale_transfers() -> None:
|
|
240
|
+
"""Delete SERVE_DIR leftovers older than ``_SERVE_TTL_SECONDS`` (rate-limited).
|
|
241
|
+
|
|
242
|
+
Covers aborts between staging and register (a file without a
|
|
243
|
+
route). Runs in the request path, but at most every
|
|
244
|
+
``_SWEEP_INTERVAL`` seconds (``_last_sweep``); every file operation
|
|
245
|
+
is best-effort.
|
|
246
|
+
"""
|
|
247
|
+
global _last_sweep
|
|
248
|
+
now = time.time()
|
|
249
|
+
if now - _last_sweep < _SWEEP_INTERVAL:
|
|
250
|
+
return
|
|
251
|
+
_last_sweep = now
|
|
252
|
+
cutoff = now - _SERVE_TTL_SECONDS
|
|
253
|
+
try:
|
|
254
|
+
entries = list(SERVE_DIR.iterdir())
|
|
255
|
+
except OSError:
|
|
256
|
+
return
|
|
257
|
+
for p in entries:
|
|
258
|
+
try:
|
|
259
|
+
if p.is_file() and p.stat().st_mtime < cutoff:
|
|
260
|
+
p.unlink(missing_ok=True)
|
|
261
|
+
except OSError:
|
|
262
|
+
continue
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def serve_port() -> int:
|
|
266
|
+
"""Serve port: env ``IOWAP_SERVE_PORT``, else ``SERVE_PORT_DEFAULT``.
|
|
267
|
+
|
|
268
|
+
Invalid env values (not an int, out of range) → WARNING + default —
|
|
269
|
+
a typo in the env does not break the daemon (house pattern like
|
|
270
|
+
``relay_client._effective_config``).
|
|
271
|
+
"""
|
|
272
|
+
raw = os.environ.get("IOWAP_SERVE_PORT")
|
|
273
|
+
if raw is None:
|
|
274
|
+
return SERVE_PORT_DEFAULT
|
|
275
|
+
try:
|
|
276
|
+
port = int(raw)
|
|
277
|
+
except ValueError:
|
|
278
|
+
log.warning("ignoring invalid IOWAP_SERVE_PORT=%r", raw)
|
|
279
|
+
return SERVE_PORT_DEFAULT
|
|
280
|
+
if not 1 <= port <= 65535:
|
|
281
|
+
log.warning("ignoring out-of-range IOWAP_SERVE_PORT=%r", raw)
|
|
282
|
+
return SERVE_PORT_DEFAULT
|
|
283
|
+
return port
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def read_manifest() -> dict | None:
|
|
287
|
+
"""Read serve.json; ``None`` when missing or invalid.
|
|
288
|
+
|
|
289
|
+
"Invalid" explicitly includes: not JSON, not a dict, ``port``
|
|
290
|
+
missing/not an int/not a valid port value. The normal case without
|
|
291
|
+
a daemon is ``None`` (the CLI then reports the F5 error).
|
|
292
|
+
|
|
293
|
+
D8: the return additionally contains ``host`` (the advertise
|
|
294
|
+
address for the upstream URL). Manifests without a ``host`` key
|
|
295
|
+
(before D8) get the default ``SERVE_HOST`` — an invalid ``host``
|
|
296
|
+
value likewise falls back to the default instead of discarding the
|
|
297
|
+
entire manifest (the port remains the relevant part of the
|
|
298
|
+
contract).
|
|
299
|
+
"""
|
|
300
|
+
try:
|
|
301
|
+
raw = MANIFEST_PATH.read_text()
|
|
302
|
+
except FileNotFoundError:
|
|
303
|
+
return None
|
|
304
|
+
except OSError as exc:
|
|
305
|
+
log.warning("cannot read serve manifest %s: %s", MANIFEST_PATH, exc)
|
|
306
|
+
return None
|
|
307
|
+
try:
|
|
308
|
+
data = json.loads(raw)
|
|
309
|
+
except ValueError as exc:
|
|
310
|
+
log.warning("serve manifest %s is not valid JSON: %s", MANIFEST_PATH, exc)
|
|
311
|
+
return None
|
|
312
|
+
if not isinstance(data, dict):
|
|
313
|
+
return None
|
|
314
|
+
port = data.get("port")
|
|
315
|
+
# bool is an int subclass — otherwise True would be "port 1".
|
|
316
|
+
if isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535:
|
|
317
|
+
return None
|
|
318
|
+
host = data.get("host", SERVE_HOST)
|
|
319
|
+
if not isinstance(host, str) or not host or len(host) > 255:
|
|
320
|
+
log.warning("serve manifest has invalid host %r — using default", host)
|
|
321
|
+
host = SERVE_HOST
|
|
322
|
+
return {"port": port, "host": host}
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def write_manifest(port: int, host: str | None = None) -> None:
|
|
326
|
+
"""Write ``{"port": port, "host": host}`` to ``MANIFEST_PATH`` — atomically.
|
|
327
|
+
|
|
328
|
+
The daemon writes the manifest at serve start (F5 port discovery);
|
|
329
|
+
a CLI reading midway through the write never sees a half-written
|
|
330
|
+
JSON. ``host`` is the D8 advertise address (``serve_host()``);
|
|
331
|
+
``None`` writes the default ``SERVE_HOST`` — old manifests without
|
|
332
|
+
a ``host`` key stay readable (read_manifest adds it, see there).
|
|
333
|
+
"""
|
|
334
|
+
MANIFEST_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
335
|
+
tmp = MANIFEST_PATH.with_name(MANIFEST_PATH.name + ".tmp")
|
|
336
|
+
tmp.write_text(
|
|
337
|
+
json.dumps({"port": port, "host": host if host is not None else SERVE_HOST})
|
|
338
|
+
)
|
|
339
|
+
os.replace(tmp, MANIFEST_PATH)
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def new_transfer_id() -> str:
|
|
343
|
+
"""``secrets.token_urlsafe(16)`` → 22 url-safe chars (F2).
|
|
344
|
+
|
|
345
|
+
22 ≤ the route registry's 64-char limit; collisions are impossible
|
|
346
|
+
with 128 bits of entropy.
|
|
347
|
+
"""
|
|
348
|
+
return secrets.token_urlsafe(16)
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def stage_file(src: Path, *, max_downloads: int = 1) -> tuple[str, Path, str]:
|
|
352
|
+
"""Stage ``src`` to ``SERVE_DIR/{transfer_id}``, return (id, staged, sha256).
|
|
353
|
+
|
|
354
|
+
Move semantics (F5): ``os.replace`` = O(1) on the same filesystem,
|
|
355
|
+
with a ``shutil.copy`` + unlink fallback across filesystem
|
|
356
|
+
boundaries (EXDEV). Writes the F6 manifest
|
|
357
|
+
``SERVE_DIR/{transfer_id}.json`` with
|
|
358
|
+
``{file, size, sha256, max_downloads}`` — the daemon counts along
|
|
359
|
+
it and cleans up after ``max_downloads`` downloads.
|
|
360
|
+
|
|
361
|
+
Raise ``ValueError`` for ``max_downloads < 1`` — a 0-download
|
|
362
|
+
transfer would be a dead route (fail fast at the producer).
|
|
363
|
+
"""
|
|
364
|
+
if max_downloads < 1:
|
|
365
|
+
raise ValueError(f"max_downloads must be >= 1, got {max_downloads}")
|
|
366
|
+
SERVE_DIR.mkdir(parents=True, exist_ok=True)
|
|
367
|
+
transfer_id = new_transfer_id()
|
|
368
|
+
staged = SERVE_DIR / transfer_id
|
|
369
|
+
try:
|
|
370
|
+
os.replace(src, staged)
|
|
371
|
+
except OSError as exc:
|
|
372
|
+
if exc.errno not in (errno.EXDEV, errno.ENOTSUP):
|
|
373
|
+
raise
|
|
374
|
+
shutil.copy(src, staged)
|
|
375
|
+
src.unlink(missing_ok=True)
|
|
376
|
+
sha_hex = _sha256_file(staged)
|
|
377
|
+
_write_transfer_manifest(staged, sha_hex, max_downloads)
|
|
378
|
+
return transfer_id, staged, sha_hex
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _sha256_file(path: Path) -> str:
|
|
382
|
+
"""sha256 of a file, chunkwise (64 KiB) — no RAM peak (F8)."""
|
|
383
|
+
digest = hashlib.sha256()
|
|
384
|
+
with path.open("rb") as f:
|
|
385
|
+
for chunk in iter(lambda: f.read(64 * 1024), b""):
|
|
386
|
+
digest.update(chunk)
|
|
387
|
+
return digest.hexdigest()
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _write_transfer_manifest(staged: Path, sha_hex: str, max_downloads: int) -> None:
|
|
391
|
+
"""F6 manifest next to the staged file (atomic via tmp+replace)."""
|
|
392
|
+
manifest = Path(str(staged) + ".json")
|
|
393
|
+
tmp = Path(str(manifest) + ".tmp")
|
|
394
|
+
payload = {
|
|
395
|
+
"file": staged.name,
|
|
396
|
+
"size": staged.stat().st_size,
|
|
397
|
+
"sha256": sha_hex,
|
|
398
|
+
"max_downloads": max_downloads,
|
|
399
|
+
}
|
|
400
|
+
tmp.write_text(json.dumps(payload))
|
|
401
|
+
os.replace(tmp, manifest)
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def build_storage_ref(node_id: str, route_path: str, expires_at: str) -> dict:
|
|
405
|
+
"""F4 — ``storage_ref`` for envelope v1 (shape FROZEN):
|
|
406
|
+
|
|
407
|
+
``{"type": "node_serve", "node_id": ..., "path": ..., "expires_at": ...}``
|
|
408
|
+
|
|
409
|
+
``parse_envelope`` returns the dict generically (T-179) — no
|
|
410
|
+
envelope change, no contract break for old receivers.
|
|
411
|
+
"""
|
|
412
|
+
return {
|
|
413
|
+
"type": "node_serve",
|
|
414
|
+
"node_id": node_id,
|
|
415
|
+
"path": route_path,
|
|
416
|
+
"expires_at": expires_at,
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def proxy_url(base_url: str, node_id: str, route_path: str) -> str:
|
|
421
|
+
"""F7 — proxy URL for the receiver's pull.
|
|
422
|
+
|
|
423
|
+
Path shape identical to ``unregister_temp_route``
|
|
424
|
+
(``relay_client.py``): ``{base_url}{_ROUTE_BASE}/{node_id}{path}``.
|
|
425
|
+
``base_url`` loses a trailing slash; ``route_path`` is guaranteed
|
|
426
|
+
its leading slash (defensively against both spellings).
|
|
427
|
+
"""
|
|
428
|
+
path = route_path if route_path.startswith("/") else "/" + route_path
|
|
429
|
+
return f"{base_url.rstrip('/')}{_ROUTE_BASE}/{node_id}{path}"
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def probe_serve(port: int, timeout: float = 2.0, host: str | None = None) -> bool:
|
|
433
|
+
"""``GET http://{host|SERVE_HOST}:{port}/health`` — ``True`` iff 200.
|
|
434
|
+
|
|
435
|
+
Everything else (connection refused, timeout, 4xx/5xx, invalid
|
|
436
|
+
response) is "unreachable" → ``False`` without raising. The CLI
|
|
437
|
+
(task 4) derives the F5 error from it. ``host`` (D8) allows probing
|
|
438
|
+
against the advertise address from the manifest; the default stays
|
|
439
|
+
``SERVE_HOST`` (localhost, plan behavior).
|
|
440
|
+
"""
|
|
441
|
+
try:
|
|
442
|
+
r = httpx.get(f"http://{host or SERVE_HOST}:{port}/health", timeout=timeout)
|
|
443
|
+
except httpx.HTTPError as exc:
|
|
444
|
+
log.debug("serve probe on port %d failed: %s", port, exc)
|
|
445
|
+
return False
|
|
446
|
+
return r.status_code == 200
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
class EphemeralServeHandler(BaseHTTPRequestHandler):
|
|
450
|
+
"""POST ``/transfer/{id}`` + GET ``/health`` (T-166, F2/F6/F7).
|
|
451
|
+
|
|
452
|
+
Contracts (FROZEN from Phase 1, now implemented):
|
|
453
|
+
|
|
454
|
+
* POST ``/transfer/{transfer_id}`` (body is ignored/drained):
|
|
455
|
+
streams the staged file chunkwise (``shutil.copyfileobj``, 64 KiB)
|
|
456
|
+
with ``Content-Length``; counts the download only once the body
|
|
457
|
+
was fully sent (HTTP 200, no abort); deletes file + manifest
|
|
458
|
+
after ``max_downloads`` (F6) and triggers the ``on_exhausted``
|
|
459
|
+
hook (the daemon wires ``unregister_after_transfer`` there);
|
|
460
|
+
aborts do NOT count → retry until the TTL. Side effect:
|
|
461
|
+
rate-limited sweep.
|
|
462
|
+
* GET ``/health`` → 200 ``{"ok": true}`` (contract for ``probe_serve``).
|
|
463
|
+
* Unknown/foreign paths or a wrong ID form → 404. GET on
|
|
464
|
+
``/transfer/...`` → 404 (POST-only channel — the proxy matches
|
|
465
|
+
the registered method exactly).
|
|
466
|
+
* Logging quiet (debug level, no stderr spam per request in the
|
|
467
|
+
daemon).
|
|
468
|
+
"""
|
|
469
|
+
|
|
470
|
+
def log_message(self, format: str, *args) -> None:
|
|
471
|
+
log.debug("serve: " + format, *args)
|
|
472
|
+
|
|
473
|
+
def _drain_body(self) -> None:
|
|
474
|
+
"""Consume the request body (F7 sends it empty; the proxy may send more)."""
|
|
475
|
+
try:
|
|
476
|
+
length = int(self.headers.get("Content-Length") or 0)
|
|
477
|
+
except ValueError:
|
|
478
|
+
length = 0
|
|
479
|
+
remaining = max(0, length)
|
|
480
|
+
while remaining > 0:
|
|
481
|
+
chunk = self.rfile.read(min(64 * 1024, remaining))
|
|
482
|
+
if not chunk:
|
|
483
|
+
break
|
|
484
|
+
remaining -= len(chunk)
|
|
485
|
+
|
|
486
|
+
def _send_json(self, code: int, body: bytes) -> None:
|
|
487
|
+
self.send_response(code)
|
|
488
|
+
self.send_header("Content-Type", "application/json")
|
|
489
|
+
self.send_header("Content-Length", str(len(body)))
|
|
490
|
+
self.end_headers()
|
|
491
|
+
self.wfile.write(body)
|
|
492
|
+
|
|
493
|
+
def _peer_allowed(self) -> bool:
|
|
494
|
+
"""D9: check the socket peer against the cached allowlist.
|
|
495
|
+
|
|
496
|
+
Cached on the server object (T-128 semantics: resolved once at
|
|
497
|
+
start, ``_allowed_peers``). Fail closed applies to the remote
|
|
498
|
+
path: no relay IP resolvable → LAN peers rejected; loopback and
|
|
499
|
+
the advertise IP stay legitimate (local probe/CLI path, D8).
|
|
500
|
+
"""
|
|
501
|
+
allowed: frozenset[str] | None = getattr(self.server, "_allow_ip", None)
|
|
502
|
+
peer = self.client_address[0]
|
|
503
|
+
if not allowed:
|
|
504
|
+
log.warning(
|
|
505
|
+
"serve allowlist unresolved — rejecting remote request from %s",
|
|
506
|
+
peer,
|
|
507
|
+
)
|
|
508
|
+
return False
|
|
509
|
+
if peer in allowed:
|
|
510
|
+
return True
|
|
511
|
+
log.warning("rejected serve request from %s (allowed: %s)", peer, sorted(allowed))
|
|
512
|
+
return False
|
|
513
|
+
|
|
514
|
+
def do_GET(self) -> None:
|
|
515
|
+
self._drain_body()
|
|
516
|
+
if not self._peer_allowed():
|
|
517
|
+
self._send_json(403, b'{"error": "forbidden"}')
|
|
518
|
+
return
|
|
519
|
+
if self.path == "/health":
|
|
520
|
+
self._send_json(200, b'{"ok": true}')
|
|
521
|
+
else:
|
|
522
|
+
self._send_json(404, b'{"error": "not found"}')
|
|
523
|
+
|
|
524
|
+
def do_POST(self) -> None:
|
|
525
|
+
self._drain_body()
|
|
526
|
+
if not self._peer_allowed():
|
|
527
|
+
self._send_json(403, b'{"error": "forbidden"}')
|
|
528
|
+
return
|
|
529
|
+
m = _TRANSFER_RE.match(self.path)
|
|
530
|
+
if not m:
|
|
531
|
+
self._send_json(404, b'{"error": "not found"}')
|
|
532
|
+
return
|
|
533
|
+
transfer_id = m.group(1)
|
|
534
|
+
staged = SERVE_DIR / transfer_id
|
|
535
|
+
manifest_file = Path(str(staged) + ".json")
|
|
536
|
+
with _transfer_lock:
|
|
537
|
+
try:
|
|
538
|
+
fh = staged.open("rb")
|
|
539
|
+
except OSError:
|
|
540
|
+
self._send_json(404, b'{"error": "unknown transfer"}')
|
|
541
|
+
return
|
|
542
|
+
size = staged.stat().st_size
|
|
543
|
+
self.send_response(200)
|
|
544
|
+
self.send_header("Content-Type", "application/octet-stream")
|
|
545
|
+
self.send_header("Content-Length", str(size))
|
|
546
|
+
self.end_headers()
|
|
547
|
+
try:
|
|
548
|
+
with fh:
|
|
549
|
+
shutil.copyfileobj(fh, self.wfile, length=64 * 1024)
|
|
550
|
+
self.wfile.flush()
|
|
551
|
+
except (BrokenPipeError, ConnectionResetError, OSError) as exc:
|
|
552
|
+
# Aborted download does NOT count (F6 — retry possible until the TTL).
|
|
553
|
+
log.warning("transfer %s aborted mid-stream: %s", transfer_id, exc)
|
|
554
|
+
return
|
|
555
|
+
self._record_download(transfer_id, staged, manifest_file)
|
|
556
|
+
|
|
557
|
+
def _record_download(
|
|
558
|
+
self, transfer_id: str, staged: Path, manifest_file: Path
|
|
559
|
+
) -> None:
|
|
560
|
+
"""Count the download; at ``max_downloads`` clean up + fire the hook (F6)."""
|
|
561
|
+
with _transfer_lock:
|
|
562
|
+
try:
|
|
563
|
+
data = json.loads(manifest_file.read_text())
|
|
564
|
+
except (OSError, ValueError):
|
|
565
|
+
# A parallel request has already exhausted it — nothing left to count.
|
|
566
|
+
return
|
|
567
|
+
downloads = int(data.get("downloads", 0)) + 1
|
|
568
|
+
max_downloads = int(data.get("max_downloads", 1))
|
|
569
|
+
if downloads >= max_downloads:
|
|
570
|
+
staged.unlink(missing_ok=True)
|
|
571
|
+
manifest_file.unlink(missing_ok=True)
|
|
572
|
+
exhausted = True
|
|
573
|
+
else:
|
|
574
|
+
data["downloads"] = downloads
|
|
575
|
+
tmp = Path(str(manifest_file) + ".tmp")
|
|
576
|
+
tmp.write_text(json.dumps(data))
|
|
577
|
+
os.replace(tmp, manifest_file)
|
|
578
|
+
exhausted = False
|
|
579
|
+
if exhausted:
|
|
580
|
+
hook = _ON_EXHAUSTED
|
|
581
|
+
if hook is not None:
|
|
582
|
+
try:
|
|
583
|
+
hook(f"/download/{transfer_id}")
|
|
584
|
+
except Exception:
|
|
585
|
+
log.exception("on_exhausted hook failed for %s", transfer_id)
|
|
586
|
+
_sweep_stale_transfers()
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def start_serve_thread() -> threading.Thread:
|
|
590
|
+
"""Start the serve thread in the daemon (F1) — bind, manifest, daemon thread.
|
|
591
|
+
|
|
592
|
+
* bind ``SERVE_HOST:serve_port()`` (``ThreadingHTTPServer``),
|
|
593
|
+
``write_manifest(port)`` after a successful bind (F5 port
|
|
594
|
+
discovery).
|
|
595
|
+
* Thread ``daemon=True`` → dies with the daemon process.
|
|
596
|
+
* ``RuntimeError`` on bind failure — the daemon wrapper catches
|
|
597
|
+
OSError/RuntimeError → WARNING, the node keeps running without
|
|
598
|
+
serve (F1: ``hp put`` bridge then reports the F5 error instead
|
|
599
|
+
of crashing).
|
|
600
|
+
* Idempotency: an already-running serve thread → the same thread
|
|
601
|
+
is returned (no double bind on re-run).
|
|
602
|
+
"""
|
|
603
|
+
global _serve_server, _serve_thread
|
|
604
|
+
existing = _serve_thread
|
|
605
|
+
if existing is not None and existing.is_alive():
|
|
606
|
+
return existing
|
|
607
|
+
port = serve_port()
|
|
608
|
+
host = serve_host()
|
|
609
|
+
# D9: resolve the peer allowlist once at bind time and cache it on
|
|
610
|
+
# the server object (T-128 semantics). Without a relay IP the serve
|
|
611
|
+
# runs fail-closed for remote peers — the local probe path stays
|
|
612
|
+
# servable.
|
|
613
|
+
allow_peers = _allowed_peers()
|
|
614
|
+
if serve_allow_ip() is None:
|
|
615
|
+
log.warning(
|
|
616
|
+
"serve allowlist: relay IP unresolved (set IOWAP_SERVE_ALLOW) "
|
|
617
|
+
"— remote serve requests will be rejected until daemon restart"
|
|
618
|
+
)
|
|
619
|
+
try:
|
|
620
|
+
server = ThreadingHTTPServer((host, port), EphemeralServeHandler)
|
|
621
|
+
except OSError as exc:
|
|
622
|
+
raise RuntimeError(
|
|
623
|
+
f"ephemeral serve bind failed on {host}:{port}: {exc}"
|
|
624
|
+
) from exc
|
|
625
|
+
server._allow_ip = allow_peers # type: ignore[attr-defined]
|
|
626
|
+
_serve_server = server
|
|
627
|
+
write_manifest(port, host=host)
|
|
628
|
+
t = threading.Thread(
|
|
629
|
+
target=server.serve_forever, daemon=True, name="file-serve"
|
|
630
|
+
)
|
|
631
|
+
t.start()
|
|
632
|
+
_serve_thread = t
|
|
633
|
+
return t
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def unregister_after_transfer(
|
|
637
|
+
client: RelayClient, route_path: str, method: str = "POST"
|
|
638
|
+
) -> None:
|
|
639
|
+
"""Best-effort route deletion after an exhausted transfer (F6).
|
|
640
|
+
|
|
641
|
+
The daemon calls this after the last successful download —
|
|
642
|
+
HTTP errors are only logged (never re-raised): the download itself
|
|
643
|
+
already succeeded; the TTL sweep (T-125) is the backstop.
|
|
644
|
+
"""
|
|
645
|
+
try:
|
|
646
|
+
client.unregister_temp_route(route_path, method=method)
|
|
647
|
+
except httpx.HTTPError as exc:
|
|
648
|
+
log.warning("route unregister failed for %s (%s): %s", route_path, method, exc)
|