nullgate 1.2.6__tar.gz → 1.2.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nullgate-1.2.6/src/nullgate.egg-info → nullgate-1.2.7}/PKG-INFO +1 -1
- nullgate-1.2.7/src/nullgate/__init__.py +1 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/src/nullgate/ingress.py +87 -32
- {nullgate-1.2.6 → nullgate-1.2.7}/src/nullgate/session.py +324 -32
- nullgate-1.2.7/src/nullgate/supervisor.py +70 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/src/nullgate/transports.py +27 -0
- {nullgate-1.2.6 → nullgate-1.2.7/src/nullgate.egg-info}/PKG-INFO +1 -1
- {nullgate-1.2.6 → nullgate-1.2.7}/src/nullgate.egg-info/SOURCES.txt +1 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/tests/test_commands.py +444 -10
- nullgate-1.2.7/tests/test_ingress.py +279 -0
- nullgate-1.2.6/src/nullgate/__init__.py +0 -1
- nullgate-1.2.6/tests/test_ingress.py +0 -119
- {nullgate-1.2.6 → nullgate-1.2.7}/LICENSE +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/README.md +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/pyproject.toml +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/setup.cfg +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/src/nullgate/account.py +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/src/nullgate/bridge.py +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/src/nullgate/client_config.py +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/src/nullgate/commands.py +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/src/nullgate/gateway.py +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/src/nullgate/runtime.py +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/src/nullgate/wsroute.py +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/src/nullgate.egg-info/dependency_links.txt +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/src/nullgate.egg-info/entry_points.txt +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/src/nullgate.egg-info/requires.txt +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/src/nullgate.egg-info/top_level.txt +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/tests/test_account.py +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/tests/test_bridge.py +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/tests/test_gateway.py +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/tests/test_install.py +0 -0
- {nullgate-1.2.6 → nullgate-1.2.7}/tests/test_wsroute.py +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.2.7"
|
|
@@ -2,9 +2,11 @@
|
|
|
2
2
|
"""srv.us reverse tunnel for the Nullgate srvus transport.
|
|
3
3
|
|
|
4
4
|
Holds one AsyncSSH connection to srv.us with a remote port forward pointing back
|
|
5
|
-
at the local SSH server
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
at the local SSH server, reconnecting with backoff whenever the server drops it
|
|
6
|
+
(srv.us documents wrapping the tunnel in a reconnect loop). AsyncSSH is already
|
|
7
|
+
required by the server, so speaking SSH in-process here removes the OpenSSH
|
|
8
|
+
client and ssh-keygen from the host's requirements; the tunnel then works on
|
|
9
|
+
stripped images that ship neither.
|
|
8
10
|
|
|
9
11
|
Launch with asyncssh available, e.g.:
|
|
10
12
|
python -m nullgate.ingress --local-port 4822 --slot 1
|
|
@@ -30,6 +32,9 @@ from nullgate.account import current_username, ensure_username_environment
|
|
|
30
32
|
|
|
31
33
|
IDENTITY_SEED_ENV = "NULLGATE_IDENTITY_SEED"
|
|
32
34
|
|
|
35
|
+
RECONNECT_MIN = 1.0
|
|
36
|
+
RECONNECT_MAX = 30.0
|
|
37
|
+
|
|
33
38
|
|
|
34
39
|
class TunnelClient(asyncssh.SSHClient):
|
|
35
40
|
"""Surface srv.us protocol messages in the tunnel log.
|
|
@@ -96,37 +101,19 @@ async def pump(reader: asyncssh.SSHReader, writer) -> None:
|
|
|
96
101
|
writer.flush()
|
|
97
102
|
|
|
98
103
|
|
|
99
|
-
async def
|
|
100
|
-
|
|
101
|
-
port: int,
|
|
104
|
+
async def serve_connection(
|
|
105
|
+
connection: asyncssh.SSHClientConnection,
|
|
102
106
|
local_host: str,
|
|
103
107
|
local_port: int,
|
|
104
108
|
slot: int,
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
ensure_username_environment(username)
|
|
109
|
-
ensure_key(key)
|
|
110
|
-
|
|
111
|
-
stopped = asyncio.Event()
|
|
112
|
-
loop = asyncio.get_running_loop()
|
|
113
|
-
for signum in (signal.SIGINT, signal.SIGTERM):
|
|
114
|
-
with contextlib.suppress(NotImplementedError):
|
|
115
|
-
loop.add_signal_handler(signum, stopped.set)
|
|
116
|
-
|
|
117
|
-
# known_hosts=None matches the connection commands this project prints: the
|
|
118
|
-
# ingress is disposable and only carries bytes, so it is not authenticated.
|
|
119
|
-
connection = await asyncssh.connect(
|
|
120
|
-
host,
|
|
121
|
-
port=port,
|
|
122
|
-
username=username,
|
|
123
|
-
client_keys=[str(key)],
|
|
124
|
-
known_hosts=None,
|
|
125
|
-
client_factory=TunnelClient,
|
|
126
|
-
keepalive_interval=30,
|
|
127
|
-
keepalive_count_max=3,
|
|
128
|
-
)
|
|
109
|
+
stopped: asyncio.Event,
|
|
110
|
+
) -> bool:
|
|
111
|
+
"""Hold one tunnel connection until it closes or a stop is requested.
|
|
129
112
|
|
|
113
|
+
Opens the session channel and the remote port forward, then waits. Returns
|
|
114
|
+
True when ``stopped`` was set (clean shutdown), False when the server
|
|
115
|
+
dropped the connection and the caller should reconnect.
|
|
116
|
+
"""
|
|
130
117
|
process = None
|
|
131
118
|
listener = None
|
|
132
119
|
tasks: list[asyncio.Task] = []
|
|
@@ -156,9 +143,9 @@ async def run(
|
|
|
156
143
|
tasks.extend((closed, requested))
|
|
157
144
|
await asyncio.wait((closed, requested), return_when=asyncio.FIRST_COMPLETED)
|
|
158
145
|
if requested.done():
|
|
159
|
-
return
|
|
146
|
+
return True
|
|
160
147
|
print("nullgate-ingress: connection closed by the server", flush=True)
|
|
161
|
-
return
|
|
148
|
+
return False
|
|
162
149
|
finally:
|
|
163
150
|
if listener is not None:
|
|
164
151
|
listener.close()
|
|
@@ -172,6 +159,74 @@ async def run(
|
|
|
172
159
|
await asyncio.gather(*tasks, return_exceptions=True)
|
|
173
160
|
|
|
174
161
|
|
|
162
|
+
async def backoff(stopped: asyncio.Event, delay: float) -> None:
|
|
163
|
+
"""Wait out a reconnect delay, returning early on a requested stop."""
|
|
164
|
+
with contextlib.suppress(asyncio.TimeoutError):
|
|
165
|
+
await asyncio.wait_for(stopped.wait(), timeout=delay)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
async def run(
|
|
169
|
+
host: str,
|
|
170
|
+
port: int,
|
|
171
|
+
local_host: str,
|
|
172
|
+
local_port: int,
|
|
173
|
+
slot: int,
|
|
174
|
+
key: Path,
|
|
175
|
+
username: str,
|
|
176
|
+
) -> int:
|
|
177
|
+
ensure_username_environment(username)
|
|
178
|
+
ensure_key(key)
|
|
179
|
+
|
|
180
|
+
stopped = asyncio.Event()
|
|
181
|
+
loop = asyncio.get_running_loop()
|
|
182
|
+
for signum in (signal.SIGINT, signal.SIGTERM):
|
|
183
|
+
with contextlib.suppress(NotImplementedError):
|
|
184
|
+
loop.add_signal_handler(signum, stopped.set)
|
|
185
|
+
|
|
186
|
+
# Retry forever: srv.us documents wrapping the tunnel in a reconnect loop,
|
|
187
|
+
# and a disposable gateway must survive transient server-side flaps.
|
|
188
|
+
delay = RECONNECT_MIN
|
|
189
|
+
attempt = 0
|
|
190
|
+
while not stopped.is_set():
|
|
191
|
+
try:
|
|
192
|
+
# known_hosts=None matches the connection commands this project
|
|
193
|
+
# prints: the ingress is disposable and only carries bytes, so it
|
|
194
|
+
# is not authenticated.
|
|
195
|
+
connection = await asyncssh.connect(
|
|
196
|
+
host,
|
|
197
|
+
port=port,
|
|
198
|
+
username=username,
|
|
199
|
+
client_keys=[str(key)],
|
|
200
|
+
known_hosts=None,
|
|
201
|
+
client_factory=TunnelClient,
|
|
202
|
+
keepalive_interval=30,
|
|
203
|
+
keepalive_count_max=3,
|
|
204
|
+
)
|
|
205
|
+
if attempt:
|
|
206
|
+
print("nullgate-ingress: reconnected", flush=True)
|
|
207
|
+
attempt = 0
|
|
208
|
+
delay = RECONNECT_MIN
|
|
209
|
+
clean = await serve_connection(
|
|
210
|
+
connection, local_host, local_port, slot, stopped
|
|
211
|
+
)
|
|
212
|
+
except (OSError, asyncssh.Error) as error:
|
|
213
|
+
print(f"nullgate-ingress: {error}", flush=True)
|
|
214
|
+
clean = False
|
|
215
|
+
if clean:
|
|
216
|
+
return 0
|
|
217
|
+
attempt += 1
|
|
218
|
+
if stopped.is_set():
|
|
219
|
+
break
|
|
220
|
+
delay = min(delay, RECONNECT_MAX)
|
|
221
|
+
print(
|
|
222
|
+
f"nullgate-ingress: reconnecting in {delay:g}s (attempt {attempt})",
|
|
223
|
+
flush=True,
|
|
224
|
+
)
|
|
225
|
+
await backoff(stopped, delay)
|
|
226
|
+
delay *= 2
|
|
227
|
+
return 0
|
|
228
|
+
|
|
229
|
+
|
|
175
230
|
def main(argv: list[str] | None = None) -> int:
|
|
176
231
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
177
232
|
parser.add_argument("--host", default="srv.us", help="tunnel host")
|
|
@@ -28,6 +28,14 @@ DEFAULT_SLOT = 1
|
|
|
28
28
|
DISTRIBUTION_URL = "https://shadowdocks.github.io/nullgate-dist"
|
|
29
29
|
INSTALLER_URL = f"{DISTRIBUTION_URL}/install.sh"
|
|
30
30
|
|
|
31
|
+
_FRESH_SRVUS_TIMEOUT = 20.0
|
|
32
|
+
_REUSE_SRVUS_TIMEOUT = 5.0
|
|
33
|
+
_SRVUS_PROBE_INTERVAL = 1.0
|
|
34
|
+
_TRANSPORT_RESTART_MAX = 3
|
|
35
|
+
_TRANSPORT_RESTART_WINDOW = 30.0
|
|
36
|
+
|
|
37
|
+
_UNSET: Any = object()
|
|
38
|
+
|
|
31
39
|
|
|
32
40
|
def _version() -> str:
|
|
33
41
|
try:
|
|
@@ -258,7 +266,7 @@ def _tail_file(path: Path, follow: bool) -> None:
|
|
|
258
266
|
return
|
|
259
267
|
|
|
260
268
|
|
|
261
|
-
def _status_payload() -> dict[str, Any]:
|
|
269
|
+
def _status_payload(public_healthy: Any = _UNSET) -> dict[str, Any]:
|
|
262
270
|
"""Operational snapshot shared by start/status JSON output."""
|
|
263
271
|
state = st.state_dir()
|
|
264
272
|
gateway_pid_file = state / "gateway.pid"
|
|
@@ -267,9 +275,11 @@ def _status_payload() -> dict[str, Any]:
|
|
|
267
275
|
transport_running = st.is_running(transport_pid_file)
|
|
268
276
|
provider = st.manifest_get("transport") or transports.DEFAULT_TRANSPORT
|
|
269
277
|
target = transports.resolve_target(provider, state)
|
|
270
|
-
public_healthy = None
|
|
271
278
|
if provider == "srvus" and target is not None:
|
|
272
|
-
public_healthy
|
|
279
|
+
if public_healthy is _UNSET:
|
|
280
|
+
public_healthy = transports.probe_ssh_tls(target.host)
|
|
281
|
+
else:
|
|
282
|
+
public_healthy = None
|
|
273
283
|
port_text = st.manifest_get("port")
|
|
274
284
|
try:
|
|
275
285
|
port = int(port_text) if port_text else None
|
|
@@ -299,9 +309,91 @@ def _status_payload() -> dict[str, Any]:
|
|
|
299
309
|
}
|
|
300
310
|
|
|
301
311
|
|
|
302
|
-
def
|
|
312
|
+
def _reuse_conflict(
|
|
313
|
+
args: Any,
|
|
314
|
+
transport: str,
|
|
315
|
+
root: Path,
|
|
316
|
+
port: int,
|
|
317
|
+
slot: int,
|
|
318
|
+
hostname: str,
|
|
319
|
+
endpoint: str,
|
|
320
|
+
current_provider: str,
|
|
321
|
+
) -> str | None:
|
|
322
|
+
"""Return a reason when explicit launch options differ from the live session."""
|
|
323
|
+
if getattr(args, "transport", None) is not None and transport != current_provider:
|
|
324
|
+
return (
|
|
325
|
+
f"Running {current_provider} session differs from requested {transport}; "
|
|
326
|
+
"stop it first"
|
|
327
|
+
)
|
|
328
|
+
if getattr(args, "workspace", None) not in (None, ""):
|
|
329
|
+
current_root = st.manifest_get("sftp_root") or st.read_settings().get("root", "")
|
|
330
|
+
if current_root and str(root) != current_root:
|
|
331
|
+
return (
|
|
332
|
+
f"Running session workspace {current_root} differs from requested {root}; "
|
|
333
|
+
"stop it first"
|
|
334
|
+
)
|
|
335
|
+
if getattr(args, "port", None) is not None or getattr(args, "port_arg", None) is not None:
|
|
336
|
+
current_port = st.manifest_get("port")
|
|
337
|
+
if current_port and str(port) != str(current_port):
|
|
338
|
+
return (
|
|
339
|
+
f"Running session port {current_port} differs from requested {port}; "
|
|
340
|
+
"stop it first"
|
|
341
|
+
)
|
|
342
|
+
if getattr(args, "slot", None) is not None or getattr(args, "slot_arg", None) is not None:
|
|
343
|
+
current_slot = st.read_settings().get("slot", "")
|
|
344
|
+
if current_slot and str(slot) != str(current_slot):
|
|
345
|
+
return (
|
|
346
|
+
f"Running session slot {current_slot} differs from requested {slot}; "
|
|
347
|
+
"stop it first"
|
|
348
|
+
)
|
|
349
|
+
if getattr(args, "hostname", None):
|
|
350
|
+
current_hostname = st.manifest_get("hostname") or st.read_settings().get(
|
|
351
|
+
"hostname", ""
|
|
352
|
+
)
|
|
353
|
+
if current_hostname and hostname != current_hostname:
|
|
354
|
+
return "Running session hostname differs from requested hostname; stop it first"
|
|
355
|
+
if getattr(args, "endpoint", None):
|
|
356
|
+
current_endpoint = st.manifest_get("endpoint") or st.read_settings().get(
|
|
357
|
+
"endpoint", ""
|
|
358
|
+
)
|
|
359
|
+
if current_endpoint and endpoint != current_endpoint:
|
|
360
|
+
return "Running session endpoint differs from requested endpoint; stop it first"
|
|
361
|
+
accept_arg = getattr(args, "accept", None)
|
|
362
|
+
if accept_arg is not None:
|
|
363
|
+
current_accept = st.manifest_get("accept")
|
|
364
|
+
wanted = "1" if bool(accept_arg) else "0"
|
|
365
|
+
if current_accept and wanted != current_accept:
|
|
366
|
+
return "Running session auth mode differs from requested mode; stop it first"
|
|
367
|
+
forwarding_arg = getattr(args, "allow_tcp_forwarding", None)
|
|
368
|
+
if forwarding_arg is not None:
|
|
369
|
+
current_forward = st.manifest_get("allow_tcp_forwarding")
|
|
370
|
+
wanted = "1" if bool(forwarding_arg) else "0"
|
|
371
|
+
if current_forward and wanted != current_forward:
|
|
372
|
+
return "Running session forwarding differs from requested mode; stop it first"
|
|
373
|
+
confine_arg = getattr(args, "confine_sftp", None)
|
|
374
|
+
if confine_arg is not None:
|
|
375
|
+
current_confine = st.manifest_get("confine_sftp") or st.read_settings().get(
|
|
376
|
+
"confine_sftp", ""
|
|
377
|
+
)
|
|
378
|
+
wanted = "1" if bool(confine_arg) else "0"
|
|
379
|
+
if current_confine and wanted != current_confine:
|
|
380
|
+
return "Running session file-transfer differs from requested mode; stop it first"
|
|
381
|
+
return None
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _public_row(color: _Color, public_healthy: bool | None) -> str | None:
|
|
385
|
+
if public_healthy is None:
|
|
386
|
+
return None
|
|
387
|
+
if public_healthy:
|
|
388
|
+
return " public " + color.green("healthy")
|
|
389
|
+
return " public " + color.red(
|
|
390
|
+
"unhealthy"
|
|
391
|
+
) + color.dim(" (route announced but TLS/SSH probe failed; retry or use --replace-stale)")
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def _show_status(args: Any = None, public_healthy: Any = _UNSET) -> int:
|
|
303
395
|
if getattr(args, "json", False):
|
|
304
|
-
payload = _status_payload()
|
|
396
|
+
payload = _status_payload(public_healthy)
|
|
305
397
|
print(json.dumps(payload, indent=2))
|
|
306
398
|
return 0 if payload["running"] and payload["public_healthy"] is not False else 1
|
|
307
399
|
state = st.state_dir()
|
|
@@ -312,9 +404,12 @@ def _show_status(args: Any = None) -> int:
|
|
|
312
404
|
transport = st.manifest_get("transport") or transports.DEFAULT_TRANSPORT
|
|
313
405
|
target = transports.resolve_target(transport, state)
|
|
314
406
|
live = bool(gateway_running and transport_running)
|
|
315
|
-
public_healthy
|
|
316
|
-
|
|
317
|
-
|
|
407
|
+
if public_healthy is _UNSET:
|
|
408
|
+
public_healthy = (
|
|
409
|
+
transports.probe_ssh_tls(target.host)
|
|
410
|
+
if transport == "srvus" and target is not None
|
|
411
|
+
else None
|
|
412
|
+
)
|
|
318
413
|
usable = live and public_healthy is not False
|
|
319
414
|
if not sys.stdout.isatty():
|
|
320
415
|
if target is not None and usable:
|
|
@@ -330,6 +425,9 @@ def _show_status(args: Any = None) -> int:
|
|
|
330
425
|
print()
|
|
331
426
|
print(f" gateway {_proc_row(color, gateway_running, st.read_pid(gateway_pid_file))}")
|
|
332
427
|
print(f" transport {_proc_row(color, transport_running, st.read_pid(transport_pid_file))}")
|
|
428
|
+
row = _public_row(color, public_healthy)
|
|
429
|
+
if row is not None:
|
|
430
|
+
print(row)
|
|
333
431
|
_print_auth_rows(color, state)
|
|
334
432
|
if target is not None and public_healthy is not False:
|
|
335
433
|
print()
|
|
@@ -358,6 +456,110 @@ def _publish_rendezvous(rendezvous_url: str, target: Any) -> None:
|
|
|
358
456
|
raise OSError(f"HTTP {response.status}")
|
|
359
457
|
|
|
360
458
|
|
|
459
|
+
def _restart_transport(transport_pid_file: Path, state: Path) -> None:
|
|
460
|
+
"""Relaunch the transport child from persisted settings.
|
|
461
|
+
|
|
462
|
+
Re-derives the spawn exactly as ``_launch`` does: same transport type,
|
|
463
|
+
port, slot, persisted session id, and identity seed, so a restarted srv.us
|
|
464
|
+
ingress keeps the same key-derived hostname.
|
|
465
|
+
"""
|
|
466
|
+
saved = st.read_settings()
|
|
467
|
+
transport = saved.get("transport") or transports.DEFAULT_TRANSPORT
|
|
468
|
+
_, port, slot = _resolve_launch_settings(None, saved)
|
|
469
|
+
_, identity_seed = _resolve_identity_seed(None, saved, transport)
|
|
470
|
+
accept = st.manifest_get("accept") == "1"
|
|
471
|
+
proc = transports.launch_transport(
|
|
472
|
+
transport=transport,
|
|
473
|
+
port=port,
|
|
474
|
+
slot=slot,
|
|
475
|
+
session=st.get_or_create_session(),
|
|
476
|
+
state=state,
|
|
477
|
+
endpoint=saved.get("endpoint") or "",
|
|
478
|
+
hostname=saved.get("hostname") or "",
|
|
479
|
+
token=saved.get("token") or "",
|
|
480
|
+
accept=accept,
|
|
481
|
+
identity_seed=identity_seed if transport == "srvus" else "",
|
|
482
|
+
)
|
|
483
|
+
if not st.write_pid(transport_pid_file, proc.pid):
|
|
484
|
+
st.kill_untracked(proc.pid)
|
|
485
|
+
raise SystemExit("Unable to track transport process")
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def _restart_gateway(gateway_pid_file: Path, state: Path) -> None:
|
|
489
|
+
"""Relaunch the gateway child from persisted settings.
|
|
490
|
+
|
|
491
|
+
Re-derives the spawn exactly as ``_launch`` does: same transport, root,
|
|
492
|
+
port, password (or accept mode), forwarding, and SFTP confinement, so the
|
|
493
|
+
replacement gateway serves the same workspace on the same port.
|
|
494
|
+
"""
|
|
495
|
+
saved = st.read_settings()
|
|
496
|
+
transport = saved.get("transport") or transports.DEFAULT_TRANSPORT
|
|
497
|
+
root, port, _ = _resolve_launch_settings(None, saved)
|
|
498
|
+
accept = st.manifest_get("accept") == "1"
|
|
499
|
+
password = _generated_password(state)
|
|
500
|
+
allow_tcp_forwarding = st.manifest_get("allow_tcp_forwarding") == "1"
|
|
501
|
+
confine = st.manifest_get("confine_sftp") or st.read_settings().get("confine_sftp", "")
|
|
502
|
+
proc = transports.launch_gateway(
|
|
503
|
+
transport=transport,
|
|
504
|
+
root=root,
|
|
505
|
+
port=port,
|
|
506
|
+
password=password,
|
|
507
|
+
accept=accept,
|
|
508
|
+
allow_tcp_forwarding=allow_tcp_forwarding,
|
|
509
|
+
confine_sftp=confine == "1",
|
|
510
|
+
state=state,
|
|
511
|
+
)
|
|
512
|
+
if not st.write_pid(gateway_pid_file, proc.pid):
|
|
513
|
+
st.kill_untracked(proc.pid)
|
|
514
|
+
raise SystemExit("Unable to track gateway process")
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def _supervise_tick(
|
|
518
|
+
gateway_pid_file: Path,
|
|
519
|
+
transport_pid_file: Path,
|
|
520
|
+
gateway_log: Path,
|
|
521
|
+
transport_log: Path,
|
|
522
|
+
restarts: dict[str, list[float]],
|
|
523
|
+
) -> str:
|
|
524
|
+
"""One supervision poll; relaunch a dead child when bounded.
|
|
525
|
+
|
|
526
|
+
Mutates ``restarts[child]`` with the restart timestamps within
|
|
527
|
+
``_TRANSPORT_RESTART_WINDOW`` and returns "none", "relaunched", or "exit"
|
|
528
|
+
(restart budget exceeded; the caller reports and leaves).
|
|
529
|
+
"""
|
|
530
|
+
child = None
|
|
531
|
+
if not st.is_running(gateway_pid_file):
|
|
532
|
+
pid_file, child = gateway_pid_file, "gateway"
|
|
533
|
+
elif not st.is_running(transport_pid_file):
|
|
534
|
+
pid_file, child = transport_pid_file, "transport"
|
|
535
|
+
if child is None:
|
|
536
|
+
return "none"
|
|
537
|
+
pid_file.unlink(missing_ok=True)
|
|
538
|
+
recent = [
|
|
539
|
+
stamp
|
|
540
|
+
for stamp in restarts[child]
|
|
541
|
+
if time.monotonic() - stamp < _TRANSPORT_RESTART_WINDOW
|
|
542
|
+
]
|
|
543
|
+
if len(recent) + 1 >= _TRANSPORT_RESTART_MAX:
|
|
544
|
+
return "exit"
|
|
545
|
+
recent.append(time.monotonic())
|
|
546
|
+
restarts[child] = recent
|
|
547
|
+
print(
|
|
548
|
+
f"nullgate-supervisor: {child} exited; relaunching (restart {len(recent)})",
|
|
549
|
+
flush=True,
|
|
550
|
+
)
|
|
551
|
+
try:
|
|
552
|
+
if child == "gateway":
|
|
553
|
+
_restart_gateway(gateway_pid_file, gateway_pid_file.parent)
|
|
554
|
+
else:
|
|
555
|
+
_restart_transport(transport_pid_file, transport_pid_file.parent)
|
|
556
|
+
except SystemExit as error:
|
|
557
|
+
print(error, file=sys.stderr)
|
|
558
|
+
return "exit"
|
|
559
|
+
print(f"nullgate-supervisor: {child} relaunched", flush=True)
|
|
560
|
+
return "relaunched"
|
|
561
|
+
|
|
562
|
+
|
|
361
563
|
def _supervise_foreground(
|
|
362
564
|
gateway_pid_file: Path,
|
|
363
565
|
transport_pid_file: Path,
|
|
@@ -366,8 +568,11 @@ def _supervise_foreground(
|
|
|
366
568
|
) -> int:
|
|
367
569
|
"""Keep the parent alive supervising both children.
|
|
368
570
|
|
|
369
|
-
|
|
370
|
-
|
|
571
|
+
A dead transport is relaunched in place (the gateway holds the served
|
|
572
|
+
root); restarts are bounded so a crash-looping transport surfaces its log
|
|
573
|
+
and exits instead of hot-looping. Returns 128+signum when SIGINT/SIGTERM
|
|
574
|
+
arrives (after stopping both children), or 1 with recent log tails on
|
|
575
|
+
stderr when the gateway dies or the transport keeps dying.
|
|
371
576
|
"""
|
|
372
577
|
stop_signum: list[int] = []
|
|
373
578
|
|
|
@@ -380,21 +585,44 @@ def _supervise_foreground(
|
|
|
380
585
|
previous[signum] = signal.signal(signum, _handle)
|
|
381
586
|
except (OSError, ValueError):
|
|
382
587
|
continue
|
|
588
|
+
restarts: dict[str, list[float]] = {"gateway": [], "transport": []}
|
|
383
589
|
try:
|
|
590
|
+
gateway_seen_dead = False
|
|
384
591
|
while True:
|
|
385
592
|
if stop_signum:
|
|
386
593
|
st.stop_one(transport_pid_file)
|
|
387
594
|
st.stop_one(gateway_pid_file)
|
|
388
595
|
return 128 + stop_signum[0]
|
|
389
|
-
if
|
|
390
|
-
|
|
391
|
-
):
|
|
596
|
+
if gateway_seen_dead or not st.is_running(gateway_pid_file):
|
|
597
|
+
gateway_seen_dead = True
|
|
392
598
|
print("A supervised process exited. Recent logs:", file=sys.stderr)
|
|
393
599
|
_tail_stderr(transport_log, 100)
|
|
394
600
|
_tail_stderr(gateway_log, 100)
|
|
395
601
|
st.stop_one(transport_pid_file)
|
|
396
602
|
st.stop_one(gateway_pid_file)
|
|
397
603
|
return 1
|
|
604
|
+
action = _supervise_tick(
|
|
605
|
+
gateway_pid_file,
|
|
606
|
+
transport_pid_file,
|
|
607
|
+
gateway_log,
|
|
608
|
+
transport_log,
|
|
609
|
+
restarts,
|
|
610
|
+
)
|
|
611
|
+
if action == "exit":
|
|
612
|
+
if not st.is_running(transport_pid_file):
|
|
613
|
+
print(
|
|
614
|
+
"Transport service keeps dying. Recent logs:",
|
|
615
|
+
file=sys.stderr,
|
|
616
|
+
)
|
|
617
|
+
_tail_stderr(transport_log, 100)
|
|
618
|
+
st.stop_one(gateway_pid_file)
|
|
619
|
+
else:
|
|
620
|
+
print("A supervised process exited. Recent logs:", file=sys.stderr)
|
|
621
|
+
_tail_stderr(transport_log, 100)
|
|
622
|
+
_tail_stderr(gateway_log, 100)
|
|
623
|
+
st.stop_one(transport_pid_file)
|
|
624
|
+
st.stop_one(gateway_pid_file)
|
|
625
|
+
return 1
|
|
398
626
|
time.sleep(0.5)
|
|
399
627
|
finally:
|
|
400
628
|
for signum, handler in previous.items():
|
|
@@ -444,16 +672,50 @@ def _launch(args: Any, saved: dict[str, Any]) -> int:
|
|
|
444
672
|
if gateway_running or transport_running:
|
|
445
673
|
current_provider = st.manifest_get("transport") or transports.DEFAULT_TRANSPORT
|
|
446
674
|
current_target = transports.resolve_target(current_provider, state)
|
|
447
|
-
publicly_healthy = True
|
|
448
675
|
if current_provider == "srvus":
|
|
449
|
-
|
|
450
|
-
|
|
676
|
+
current_host = current_target.host if current_target is not None else ""
|
|
677
|
+
if current_host:
|
|
678
|
+
publicly_healthy: bool | None = transports.wait_for_ssh_tls(
|
|
679
|
+
current_host,
|
|
680
|
+
timeout=_REUSE_SRVUS_TIMEOUT,
|
|
681
|
+
interval=_SRVUS_PROBE_INTERVAL,
|
|
682
|
+
)
|
|
683
|
+
else:
|
|
684
|
+
publicly_healthy = False
|
|
685
|
+
else:
|
|
686
|
+
publicly_healthy = None
|
|
687
|
+
live = bool(gateway_running and transport_running)
|
|
688
|
+
if live and publicly_healthy is not False:
|
|
689
|
+
conflict = _reuse_conflict(
|
|
690
|
+
args,
|
|
691
|
+
transport,
|
|
692
|
+
root,
|
|
693
|
+
port,
|
|
694
|
+
slot,
|
|
695
|
+
hostname,
|
|
696
|
+
endpoint,
|
|
697
|
+
current_provider,
|
|
451
698
|
)
|
|
452
|
-
|
|
699
|
+
if conflict is not None:
|
|
700
|
+
die(conflict)
|
|
701
|
+
if not getattr(args, "replace_stale", False):
|
|
702
|
+
if gateway_running:
|
|
703
|
+
die("Gateway service is already running")
|
|
704
|
+
die("Transport service is already running")
|
|
705
|
+
if current_provider == "srvus" and rendezvous_url and current_target is not None:
|
|
706
|
+
try:
|
|
707
|
+
_publish_rendezvous(rendezvous_url, current_target)
|
|
708
|
+
except Exception:
|
|
709
|
+
print(
|
|
710
|
+
"Rendezvous readiness publication failed. Recent logs:",
|
|
711
|
+
file=sys.stderr,
|
|
712
|
+
)
|
|
713
|
+
_tail_stderr(state / "transport.log", 100)
|
|
714
|
+
return 1
|
|
453
715
|
if getattr(args, "json", False):
|
|
454
|
-
print(json.dumps(_status_payload(), indent=2))
|
|
716
|
+
print(json.dumps(_status_payload(publicly_healthy), indent=2))
|
|
455
717
|
else:
|
|
456
|
-
_show_status()
|
|
718
|
+
_show_status(args, publicly_healthy)
|
|
457
719
|
return 0
|
|
458
720
|
if getattr(args, "replace_stale", False):
|
|
459
721
|
st.stop_one(transport_pid_file)
|
|
@@ -578,10 +840,19 @@ def _launch(args: Any, saved: dict[str, Any]) -> int:
|
|
|
578
840
|
)
|
|
579
841
|
_tail_stderr(transport_log, 100)
|
|
580
842
|
return 1
|
|
581
|
-
if not transports.
|
|
843
|
+
if not transports.wait_for_ssh_tls(
|
|
844
|
+
host,
|
|
845
|
+
timeout=_FRESH_SRVUS_TIMEOUT,
|
|
846
|
+
interval=_SRVUS_PROBE_INTERVAL,
|
|
847
|
+
):
|
|
582
848
|
st.stop_one(transport_pid_file)
|
|
583
849
|
st.stop_one(gateway_pid_file)
|
|
584
|
-
print(
|
|
850
|
+
print(
|
|
851
|
+
"srv.us public endpoint failed its SSH-over-TLS probe. "
|
|
852
|
+
"Recent logs:",
|
|
853
|
+
file=sys.stderr,
|
|
854
|
+
)
|
|
855
|
+
_tail_stderr(transport_log, 100)
|
|
585
856
|
return 1
|
|
586
857
|
elif transport == "upterm":
|
|
587
858
|
if not transports.wait_for_upterm_session(transport_pid_file, state / "upterm.json"):
|
|
@@ -613,7 +884,8 @@ def _launch(args: Any, saved: dict[str, Any]) -> int:
|
|
|
613
884
|
except Exception:
|
|
614
885
|
st.stop_one(transport_pid_file)
|
|
615
886
|
st.stop_one(gateway_pid_file)
|
|
616
|
-
print("Rendezvous readiness publication failed.", file=sys.stderr)
|
|
887
|
+
print("Rendezvous readiness publication failed. Recent logs:", file=sys.stderr)
|
|
888
|
+
_tail_stderr(transport_log, 100)
|
|
617
889
|
return 1
|
|
618
890
|
|
|
619
891
|
manifest: dict[str, Any] = {
|
|
@@ -657,12 +929,13 @@ def _launch(args: Any, saved: dict[str, Any]) -> int:
|
|
|
657
929
|
|
|
658
930
|
json_mode = bool(getattr(args, "json", False))
|
|
659
931
|
foreground = bool(getattr(args, "foreground", False))
|
|
932
|
+
established: bool | None = True if transport == "srvus" else None
|
|
660
933
|
if foreground:
|
|
661
934
|
if json_mode:
|
|
662
|
-
print(json.dumps(_status_payload(), indent=2))
|
|
935
|
+
print(json.dumps(_status_payload(established), indent=2))
|
|
663
936
|
sys.stdout.flush()
|
|
664
937
|
else:
|
|
665
|
-
_show_status()
|
|
938
|
+
_show_status(args, established)
|
|
666
939
|
try:
|
|
667
940
|
return _supervise_foreground(
|
|
668
941
|
gateway_pid_file, transport_pid_file, gateway_log, transport_log
|
|
@@ -671,10 +944,21 @@ def _launch(args: Any, saved: dict[str, Any]) -> int:
|
|
|
671
944
|
st.stop_one(transport_pid_file)
|
|
672
945
|
st.stop_one(gateway_pid_file)
|
|
673
946
|
raise
|
|
947
|
+
supervisor_pid_file = state / "supervisor.pid"
|
|
948
|
+
supervisor_proc = transports._spawn(
|
|
949
|
+
[sys.executable, "-m", "nullgate.supervisor"],
|
|
950
|
+
transports._scrubbed_env(),
|
|
951
|
+
state / "supervisor.log",
|
|
952
|
+
)
|
|
953
|
+
if not st.write_pid(supervisor_pid_file, supervisor_proc.pid):
|
|
954
|
+
st.kill_untracked(supervisor_proc.pid)
|
|
955
|
+
die("Unable to track supervisor process")
|
|
674
956
|
if json_mode:
|
|
675
|
-
print(json.dumps(_status_payload(), indent=2))
|
|
957
|
+
print(json.dumps(_status_payload(established), indent=2))
|
|
676
958
|
else:
|
|
677
|
-
|
|
959
|
+
if sys.stdout.isatty():
|
|
960
|
+
print("Supervisor active: gateway and transport are kept alive in the background.")
|
|
961
|
+
_show_status(args, established)
|
|
678
962
|
return 0
|
|
679
963
|
|
|
680
964
|
|
|
@@ -688,6 +972,7 @@ def cmd_shut(args: Any = None) -> int:
|
|
|
688
972
|
del args
|
|
689
973
|
st.setup_state()
|
|
690
974
|
state = st.state_dir()
|
|
975
|
+
st.stop_one(state / "supervisor.pid")
|
|
691
976
|
st.stop_one(state / "transport.pid")
|
|
692
977
|
st.stop_one(state / "gateway.pid")
|
|
693
978
|
(state / "manifest.json").unlink(missing_ok=True)
|
|
@@ -705,9 +990,17 @@ def cmd_enter(args: Any = None) -> int:
|
|
|
705
990
|
state = st.state_dir()
|
|
706
991
|
transport = st.manifest_get("transport") or transports.DEFAULT_TRANSPORT
|
|
707
992
|
target = transports.resolve_target(transport, state)
|
|
993
|
+
public_healthy = None
|
|
994
|
+
if transport == "srvus" and target is not None:
|
|
995
|
+
public_healthy = transports.probe_ssh_tls(target.host)
|
|
708
996
|
if getattr(args, "json", False):
|
|
709
|
-
payload = _status_payload()
|
|
710
|
-
if target is None:
|
|
997
|
+
payload = _status_payload(public_healthy)
|
|
998
|
+
if target is None or public_healthy is False:
|
|
999
|
+
note = (
|
|
1000
|
+
"No publicly healthy route; check `nullgate status`."
|
|
1001
|
+
if public_healthy is False
|
|
1002
|
+
else "No published address yet; check `nullgate status`."
|
|
1003
|
+
)
|
|
711
1004
|
payload.update(
|
|
712
1005
|
{
|
|
713
1006
|
"url": None,
|
|
@@ -719,10 +1012,7 @@ def cmd_enter(args: Any = None) -> int:
|
|
|
719
1012
|
}
|
|
720
1013
|
)
|
|
721
1014
|
print(json.dumps(payload, indent=2))
|
|
722
|
-
print(
|
|
723
|
-
"No published address yet; check `nullgate status`.",
|
|
724
|
-
file=sys.stderr,
|
|
725
|
-
)
|
|
1015
|
+
print(note, file=sys.stderr)
|
|
726
1016
|
return 1
|
|
727
1017
|
payload.update(
|
|
728
1018
|
{
|
|
@@ -734,6 +1024,8 @@ def cmd_enter(args: Any = None) -> int:
|
|
|
734
1024
|
return 0
|
|
735
1025
|
if target is None:
|
|
736
1026
|
die("No published address yet; check `nullgate status`.")
|
|
1027
|
+
if public_healthy is False:
|
|
1028
|
+
die("No publicly healthy route; check `nullgate status`.")
|
|
737
1029
|
if not sys.stdout.isatty():
|
|
738
1030
|
_print_compact_connection(state, target)
|
|
739
1031
|
return 0
|