lablink-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1203 @@
1
+ """`lablink deploy/destroy` — manual-provider compose orchestration.
2
+
3
+ The allocator image is monolithic: it bundles Flask + nginx + an internal
4
+ Postgres. This module renders a single-service docker-compose stack
5
+ (plus a `.env` and `config.yaml`) into a per-deployment workdir under
6
+ `~/.lablink/compose/<deployment_name>/`, runs `docker compose up -d`,
7
+ polls the allocator's `/api/health` endpoint, then prints a summary
8
+ including the register-token that BYO clients use to join.
9
+
10
+ Admin/DB credentials live inside the rendered `config.yaml` (mounted at
11
+ `/config/config.yaml`), not in env vars — the allocator container does
12
+ not read those from the environment.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import re
18
+ import shutil
19
+ import socket
20
+ import time
21
+ from datetime import datetime, timezone
22
+ from importlib import resources
23
+ from pathlib import Path
24
+
25
+ import typer
26
+ from rich.console import Console
27
+
28
+ from lablink_cli.commands.status import check_health_endpoint
29
+ from lablink_cli.commands.utils import resolve_admin_credentials
30
+ from lablink_cli.config.schema import Config, save_config
31
+ from lablink_cli.deployment_metrics import (
32
+ DeploymentMetrics,
33
+ cache_path_for,
34
+ phase_timer,
35
+ write_metrics,
36
+ )
37
+ from lablink_cli.docker import Docker, DockerUnavailable, default_docker
38
+
39
+ DEFAULT_COMPOSE_DIR = Path.home() / ".lablink" / "compose"
40
+ DEFAULT_HTTP_PORT = "80"
41
+ HEALTH_POLL_TIMEOUT_SECONDS = 300
42
+ ALLOCATOR_IMAGE_BASE = "ghcr.io/talmolab/lablink-allocator-image"
43
+ # Only ssl=none is supported by the manual-provider compose stack today:
44
+ # the allocator image has no TLS terminator (Caddy is part of the AWS
45
+ # infrastructure, not the container). For public TLS, operators front the
46
+ # stack with their own reverse proxy.
47
+ SUPPORTED_SSL_FOR_MANUAL = ("none",)
48
+ ALLOCATOR_CONTAINER_NAME = "lablink-allocator"
49
+ TAILSCALE_SIDECAR_CONTAINER_NAME = "lablink-allocator-tailscale"
50
+ # The allocator's own nginx port inside the sidecar's shared network
51
+ # namespace — same target the manual `tailscale funnel 5000` spike used.
52
+ ALLOCATOR_INTERNAL_PORT = 5000
53
+ # Exact substring from `tailscale funnel`'s own output when the tailnet
54
+ # hasn't granted the Funnel ACL yet (verified live, 2026-07-22 spike).
55
+ FUNNEL_ACL_NOT_GRANTED_MARKER = "Funnel is not enabled on your tailnet"
56
+ FUNNEL_ENABLE_MAX_ATTEMPTS = 5
57
+ FUNNEL_ENABLE_RETRY_DELAY_SECONDS = 2
58
+ # Budget for the public-hostname check to survive cloudflared's own startup
59
+ # (see _verify_public_hostname). Live runs registered the first edge
60
+ # connection ~1s after `docker compose up` returned and served traffic within
61
+ # a few seconds; 6 tries 5s apart leaves ~25s of headroom over that.
62
+ PUBLIC_HOSTNAME_MAX_ATTEMPTS = 6
63
+ PUBLIC_HOSTNAME_RETRY_DELAY_SECONDS = 5
64
+ # Name of the file carrying the allocator's real public URL, staged next to
65
+ # config.yaml and bind-mounted to /config/<name>. Must stay in sync with
66
+ # config_helpers.CANONICAL_URL_FILENAME in the allocator package — duplicated
67
+ # rather than imported because each package's CI job installs only its own
68
+ # dependencies, so a cross-package import would fail there. Guarded by
69
+ # test_deploy_compose.py::TestCanonicalUrlFile::test_filename_matches_allocator.
70
+ CANONICAL_URL_FILENAME = "allocator-url"
71
+
72
+ console = Console()
73
+
74
+
75
+ def compose_workdir(cfg: Config, root: Path | None = None) -> Path:
76
+ """Path to the rendered compose working directory for this deployment.
77
+
78
+ `root` overrides `DEFAULT_COMPOSE_DIR` (used by tests via `workdir_root`).
79
+ """
80
+ name = cfg.deployment_name or "lablink"
81
+ return (root or DEFAULT_COMPOSE_DIR) / name
82
+
83
+
84
+ def _read_env_value(env_path: Path, key: str) -> str | None:
85
+ """Read a single KEY=value line from an existing .env file.
86
+
87
+ Used to carry TS_AUTHKEY forward across redeploys without requiring
88
+ the admin to re-supply --tailscale-authkey every time — tailscaled's
89
+ own state (the tailscale_state volume) is what actually matters after
90
+ the first join, but the sidecar's compose environment still needs
91
+ *some* value on every render.
92
+ """
93
+ if not env_path.exists():
94
+ return None
95
+ prefix = f"{key}="
96
+ for line in env_path.read_text().splitlines():
97
+ if line.startswith(prefix):
98
+ return line[len(prefix) :]
99
+ return None
100
+
101
+
102
+ def _needs_tailscale_sidecar(cfg: Config) -> bool:
103
+ """True if a tailnet join is needed for either of two independent
104
+ reasons: reaching mesh-overlay clients, or publishing the allocator
105
+ itself to participants via Funnel. Both reuse the same sidecar."""
106
+ return (
107
+ cfg.manual.connectivity == "mesh_overlay"
108
+ or cfg.manual.participant_exposure == "tailscale_funnel"
109
+ )
110
+
111
+
112
+ def _tailscale_state_volume_exists(target: Path, *, docker: Docker) -> bool:
113
+ """True if this deployment's `tailscale_state` volume already exists.
114
+
115
+ Default `lablink destroy` preserves this volume (it carries the
116
+ sidecar's Tailscale node identity, not "data") but removes the whole
117
+ working directory, including the `.env` that would otherwise carry
118
+ TS_AUTHKEY forward. Without this check, a redeploy after such a
119
+ destroy would demand a fresh --tailscale-authkey purely because
120
+ there's no .env to read one from — even though the sidecar's identity
121
+ is already authenticated and sitting in this preserved volume, and
122
+ containerboot skips the `tailscale up --authkey` step when valid
123
+ state is already present. Guessed the same way as
124
+ `_pgdata_volume_name`'s fallback (verified via `docker volume
125
+ inspect`, safe because target.name is regex-constrained to Compose's
126
+ own project-name character set).
127
+ """
128
+ return docker.volume_exists(f"{target.name}_tailscale_state")
129
+
130
+
131
+ def render_compose_dir(
132
+ cfg: Config,
133
+ target: Path,
134
+ *,
135
+ tailscale_authkey: str | None = None,
136
+ cloudflare_tunnel_token: str | None = None,
137
+ ) -> None:
138
+ """Render docker-compose.yml + .env + config.yaml into target.
139
+
140
+ The allocator image is monolithic (bundles its own Postgres), so the
141
+ compose stack is single-service — plus a `tailscale` sidecar service
142
+ whenever a tailnet join is needed for either of two independent
143
+ reasons: `cfg.manual.connectivity == "mesh_overlay"` (network_mode:
144
+ service:allocator, so the allocator's own nginx can route to a
145
+ mesh-overlay client's Tailscale hostname) or
146
+ `cfg.manual.participant_exposure == "tailscale_funnel"` (so the
147
+ allocator can publish itself to participants via Funnel). Both reuse
148
+ the exact same sidecar — it doesn't care which reason applies, which
149
+ is why it ships as one `docker-compose.override.yml` layered over the
150
+ single base stack rather than as a second full copy of it. The
151
+ internal Postgres data is persisted via a named volume on
152
+ /var/lib/postgresql. Admin/DB creds live in the saved config.yaml
153
+ (NOT in env vars) — the caller is responsible for populating
154
+ cfg.app.admin_user/admin_password (via `resolve_admin_credentials`)
155
+ before invoking this helper.
156
+
157
+ `tailscale_authkey` is only meaningful when the sidecar is needed. It
158
+ is not persisted in config.yaml (unlike admin/DB creds) — only into
159
+ this deployment's .env, and only for as long as the sidecar needs it
160
+ to join for the first time.
161
+
162
+ `cloudflare_tunnel_token` is only meaningful when
163
+ `cfg.manual.participant_exposure == "cloudflare_tunnel"`, and follows
164
+ the same rules as `tailscale_authkey`: .env only, carried forward on a
165
+ redeploy that omits it, overridden when supplied again (rotation).
166
+ """
167
+ target.mkdir(parents=True, exist_ok=True)
168
+ needs_sidecar = _needs_tailscale_sidecar(cfg)
169
+
170
+ # 1. Copy the bundled compose templates. The base stack is always the
171
+ # same file; the Tailscale sidecar arrives as Compose's own
172
+ # auto-loaded `docker-compose.override.yml`, so no `docker compose`
173
+ # call site needs `-f` flags. Delete a stale override when the
174
+ # sidecar is no longer needed — Compose would otherwise keep
175
+ # merging it and silently rejoin the tailnet on the next redeploy.
176
+ templates = resources.files("lablink_cli.templates")
177
+ (target / "docker-compose.yml").write_text(
178
+ templates.joinpath("docker-compose.yml").read_text()
179
+ )
180
+ override_path = target / "docker-compose.override.yml"
181
+ if needs_sidecar:
182
+ override_path.write_text(
183
+ templates.joinpath("docker-compose.tailscale-override.yml").read_text()
184
+ )
185
+ else:
186
+ override_path.unlink(missing_ok=True)
187
+
188
+ # 2. Render .env — only the values the compose template substitutes.
189
+ # No DB or admin creds here: they're inside config.yaml. Read the
190
+ # OLD .env (if any) before overwriting it, so a redeploy that
191
+ # omits --tailscale-authkey carries the previous value forward
192
+ # instead of blanking out an already-joined sidecar's key.
193
+ env_path = target / ".env"
194
+ previous_authkey = _read_env_value(env_path, "TS_AUTHKEY")
195
+ previous_cf_token = _read_env_value(env_path, "CLOUDFLARE_TUNNEL_TOKEN")
196
+
197
+ allocator_image = _allocator_image(cfg)
198
+ env_lines = [
199
+ f"ALLOCATOR_IMAGE={allocator_image}",
200
+ f"HTTP_PORT={DEFAULT_HTTP_PORT}",
201
+ # Always declared: the compose templates substitute it
202
+ # unconditionally, and an unset variable makes `docker compose up`
203
+ # warn on every deploy.
204
+ f"PARTICIPANT_EXPOSURE={cfg.manual.participant_exposure}",
205
+ ]
206
+ if cfg.manual.participant_exposure == "cloudflare_tunnel":
207
+ # Same carry-forward rule as TS_AUTHKEY: a redeploy that omits the
208
+ # flag keeps the working token, while an explicitly supplied one
209
+ # wins (that is the rotation path).
210
+ env_lines.append(
211
+ f"CLOUDFLARE_TUNNEL_TOKEN="
212
+ f"{cloudflare_tunnel_token or previous_cf_token or ''}"
213
+ )
214
+ if needs_sidecar:
215
+ resolved_authkey = tailscale_authkey or previous_authkey or ""
216
+ env_lines.append(f"TS_AUTHKEY={resolved_authkey}")
217
+ env_lines.append(
218
+ f"TAILSCALE_HOSTNAME=lablink-allocator-{cfg.deployment_name or 'lablink'}"
219
+ )
220
+ env_path.write_text("\n".join(env_lines) + "\n")
221
+ env_path.chmod(0o600)
222
+
223
+ # 3. Save config.yaml in the working dir. Mounted into the allocator
224
+ # container at /config/config.yaml (which matches the container's
225
+ # CONFIG_DIR default).
226
+ save_config(cfg, target / "config.yaml")
227
+
228
+ # 4. Stage the custom startup script. Mirrors deploy.py:99-117 for
229
+ # the AWS path: ~/.lablink/custom-startup.sh wins (CLI override),
230
+ # else cfg.startup_script.path on the operator's filesystem. The
231
+ # file is always materialized (empty when disabled or absent) so
232
+ # the docker-compose bind mount resolves on every deploy; the
233
+ # allocator's registration handler only forwards it to clients
234
+ # when cfg.startup_script.enabled is true AND the file is non-
235
+ # empty.
236
+ # 4b. Stage the canonical-URL file. Always materialized (empty when the
237
+ # deployment isn't Funnel-exposed) so the compose bind mount resolves
238
+ # on every deploy — same reason custom-startup.sh below always exists.
239
+ # _enable_funnel fills it in after `compose up`, since Funnel can only
240
+ # be turned on once the sidecar is running. An existing value is
241
+ # preserved across a redeploy that stays Funnel-exposed, so the window
242
+ # between container start and _enable_funnel doesn't fall back to
243
+ # request.host_url; a deployment that turns exposure off is cleared,
244
+ # which is what stops a stale public URL being handed to clients.
245
+ canonical_target = target / CANONICAL_URL_FILENAME
246
+ if cfg.manual.participant_exposure == "tailscale_funnel":
247
+ previous_url = (
248
+ canonical_target.read_text() if canonical_target.exists() else ""
249
+ )
250
+ canonical_target.write_text(previous_url)
251
+ elif cfg.manual.participant_exposure == "cloudflare_tunnel":
252
+ # Known up front — the admin typed it. No after-the-fact write, and
253
+ # no window where the allocator reports the wrong URL.
254
+ canonical_target.write_text(f"https://{cfg.manual.public_hostname}")
255
+ else:
256
+ canonical_target.write_text("")
257
+
258
+ startup_target = target / "custom-startup.sh"
259
+ if cfg.startup_script.enabled and cfg.startup_script.path:
260
+ user_script = Path.home() / ".lablink" / "custom-startup.sh"
261
+ if user_script.exists():
262
+ src_startup = user_script
263
+ else:
264
+ src_startup = Path(cfg.startup_script.path)
265
+ if src_startup.exists():
266
+ shutil.copy2(src_startup, startup_target)
267
+ else:
268
+ console.print(
269
+ f"[yellow]startup_script.enabled=true but {src_startup} "
270
+ "not found — continuing without it.[/yellow]"
271
+ )
272
+ startup_target.touch()
273
+ else:
274
+ startup_target.touch()
275
+
276
+
277
+ def _allocator_image(cfg: Config) -> str:
278
+ """Construct the full allocator image string from base + image_tag.
279
+
280
+ The canonical config exposes only image_tag (e.g.,
281
+ "linux-amd64-latest"); the registry/repo is fixed for now.
282
+ """
283
+ tag = getattr(cfg.allocator, "image_tag", None) or "linux-amd64-latest"
284
+ return f"{ALLOCATOR_IMAGE_BASE}:{tag}"
285
+
286
+
287
+ def run_deploy_compose(
288
+ cfg: Config,
289
+ *,
290
+ yes: bool = False,
291
+ workdir_root: Path | None = None,
292
+ tailscale_authkey: str | None = None,
293
+ cloudflare_tunnel_token: str | None = None,
294
+ docker: Docker | None = None,
295
+ ) -> None:
296
+ """Bring up the allocator stack via docker-compose.
297
+
298
+ Renders the compose working directory (`compose_workdir(cfg)`),
299
+ runs `docker compose up -d`, polls the allocator's `/api/health`
300
+ endpoint until it reports healthy (or times out), and prints a
301
+ summary including the register-token used by BYO clients.
302
+
303
+ `yes=True` skips the interactive confirmation prompt.
304
+ `workdir_root` overrides `DEFAULT_COMPOSE_DIR` (used by tests).
305
+ `tailscale_authkey` is required when a tailnet join is needed for
306
+ either `cfg.manual.connectivity == "mesh_overlay"` or
307
+ `cfg.manual.participant_exposure == "tailscale_funnel"`, unless a
308
+ value is already on record in this deployment's existing `.env`
309
+ (carried forward on ordinary redeploys by `render_compose_dir`) or
310
+ the sidecar already has a valid, authenticated identity sitting in
311
+ its preserved `tailscale_state` volume (e.g. after a default
312
+ `lablink destroy`, which wipes the working directory — including
313
+ `.env` — but keeps that volume specifically so this doesn't force a
314
+ needless re-auth).
315
+ `cloudflare_tunnel_token` is required when
316
+ `cfg.manual.participant_exposure == "cloudflare_tunnel"`, unless a
317
+ value is already on record in this deployment's existing `.env`. There
318
+ is no state-volume equivalent here: the tunnel's identity lives in
319
+ Cloudflare's account, and the token is the only local copy.
320
+ """
321
+ docker = docker or default_docker()
322
+ target = compose_workdir(cfg, workdir_root)
323
+
324
+ needs_sidecar = _needs_tailscale_sidecar(cfg)
325
+ if needs_sidecar:
326
+ # Checking ".env exists" alone (i.e. "is this a redeploy") isn't
327
+ # enough: a redeploy that *switches* to needing the sidecar has
328
+ # an existing .env, but that .env has no TS_AUTHKEY line to carry
329
+ # forward. Read the actual prior value (if any) so that case
330
+ # still requires --tailscale-authkey instead of silently
331
+ # rendering an empty key.
332
+ previous_authkey = _read_env_value(target / ".env", "TS_AUTHKEY")
333
+ if (
334
+ not tailscale_authkey
335
+ and not previous_authkey
336
+ and not _tailscale_state_volume_exists(target, docker=docker)
337
+ ):
338
+ console.print(
339
+ "[red]A Tailscale sidecar is needed (manual.connectivity "
340
+ "is 'mesh_overlay' and/or manual.participant_exposure is "
341
+ "'tailscale_funnel') but no --tailscale-authkey was given, "
342
+ "and no previous value is on record for this "
343
+ "deployment.[/red]\n"
344
+ "Generate an authkey from your Tailscale admin console "
345
+ "and re-run with --tailscale-authkey <key>."
346
+ )
347
+ raise SystemExit(1)
348
+
349
+ # Preflight: cloudflare_tunnel needs a hostname and a token. The
350
+ # hostname is also checked by get_config_errors(), but `lablink deploy`
351
+ # never calls that validator for the manual provider — this is the
352
+ # actual enforcement point for a hand-edited config.yaml. Modeled on
353
+ # the TS_AUTHKEY check above, minus the state-volume clause: there is
354
+ # no volume here, so "on record in .env" is the whole condition.
355
+ if cfg.manual.participant_exposure == "cloudflare_tunnel":
356
+ if not cfg.manual.public_hostname:
357
+ console.print(
358
+ "[red]manual.participant_exposure is 'cloudflare_tunnel' but "
359
+ "manual.public_hostname is empty.[/red]\n"
360
+ "Set it to the hostname you configured as the tunnel's "
361
+ "public hostname in Cloudflare (e.g. lab.smithlab.org)."
362
+ )
363
+ raise SystemExit(1)
364
+ from lablink_allocator_service.validate_config import (
365
+ PUBLIC_HOSTNAME_HINT,
366
+ is_valid_public_hostname,
367
+ )
368
+
369
+ # The value is interpolated into "https://{host}" for the canonical-URL
370
+ # file clients are handed, and canonical_base_url accepts anything that
371
+ # merely startswith("https://") — so a pasted scheme yields
372
+ # "https://https://host" that fails silently rather than loudly.
373
+ if not is_valid_public_hostname(cfg.manual.public_hostname):
374
+ console.print(
375
+ "[red]manual.public_hostname is not a bare hostname.[/red]\n"
376
+ f"It {PUBLIC_HOSTNAME_HINT}.",
377
+ highlight=False,
378
+ )
379
+ console.print(f" got: {cfg.manual.public_hostname!r}", highlight=False)
380
+ raise SystemExit(1)
381
+ previous_cf_token = _read_env_value(target / ".env", "CLOUDFLARE_TUNNEL_TOKEN")
382
+ if not cloudflare_tunnel_token and not previous_cf_token:
383
+ console.print(
384
+ "[red]manual.participant_exposure is 'cloudflare_tunnel' but "
385
+ "no --cloudflare-tunnel-token was given, and no previous "
386
+ "value is on record for this deployment.[/red]\n"
387
+ "Create a tunnel in Cloudflare's Zero Trust dashboard "
388
+ "(Networks > Tunnels), copy the token from its Docker "
389
+ "install command, and re-run with "
390
+ "--cloudflare-tunnel-token <token>."
391
+ )
392
+ raise SystemExit(1)
393
+
394
+ # Preflight: SSL provider must be one the compose template supports.
395
+ # The allocator image has no TLS terminator, so only ssl=none works
396
+ # out of the box. Operators who need TLS run their own reverse proxy
397
+ # in front of the compose stack.
398
+ if cfg.ssl.provider not in SUPPORTED_SSL_FOR_MANUAL:
399
+ console.print(
400
+ f"[red]Manual provider deploy supports only "
401
+ f"ssl.provider='none' (got '{cfg.ssl.provider}').[/red]\n"
402
+ "The allocator image has no TLS terminator; for public TLS, "
403
+ "front the compose stack with your own reverse proxy "
404
+ "(Caddy, nginx, Cloudflare Tunnel)."
405
+ )
406
+ raise SystemExit(1)
407
+
408
+ # Preflight: lan_direct + any public exposure is not a supported
409
+ # combination. lan_direct sends the participant's browser straight to
410
+ # a client's LAN IP (ws://<client-ip>:6080 — see
411
+ # LANDirectClientConnectivity), bypassing the allocator entirely —
412
+ # unreachable off-LAN and blocked as mixed content once the allocator
413
+ # itself is publicly exposed. mesh_overlay proxies sessions through the
414
+ # allocator's own nginx instead, which any exposure mode publishes.
415
+ # get_config_errors() also rejects this (catches it in the wizard/
416
+ # `show-config`/`doctor`), but `lablink deploy` never calls that
417
+ # validator for the manual provider — this is the actual enforcement
418
+ # point for a hand-edited config.yaml deployed directly.
419
+ if (
420
+ cfg.manual.participant_exposure != "none"
421
+ and cfg.manual.connectivity == "lan_direct"
422
+ ):
423
+ console.print(
424
+ f"[red]manual.participant_exposure is "
425
+ f"'{cfg.manual.participant_exposure}' but manual.connectivity is "
426
+ f"'lan_direct'.[/red]\n"
427
+ "Participant sessions would connect directly to a client's LAN "
428
+ "IP, which is unreachable off-LAN and blocked as mixed content "
429
+ "from the HTTPS page. Use manual.connectivity: mesh_overlay "
430
+ "instead, which proxies sessions through the allocator."
431
+ )
432
+ raise SystemExit(1)
433
+
434
+ # Preflight: docker on PATH.
435
+ try:
436
+ docker.require()
437
+ except DockerUnavailable:
438
+ console.print(
439
+ "[red]docker not found on PATH.[/red] "
440
+ "Install Docker Engine + the Compose plugin "
441
+ "(https://docs.docker.com/engine/install/) and re-run."
442
+ )
443
+ raise SystemExit(1)
444
+
445
+ # Resolve admin credentials (mirrors AWS deploy.py). The wizard does
446
+ # NOT collect admin user/password — they're resolved here. Write the
447
+ # resolved values back to cfg so render_compose_dir picks them up
448
+ # via cfg.app.admin_user / cfg.app.admin_password.
449
+ admin_user, admin_pw = resolve_admin_credentials(cfg)
450
+ cfg.app.admin_user = admin_user
451
+ cfg.app.admin_password = admin_pw
452
+
453
+ # Preflight: a publicly exposed allocator is scanned by bots within
454
+ # minutes of publication (empirically confirmed 2026-07-22) — refuse
455
+ # to ship a weak/example admin password once that's the case. Placed
456
+ # after resolve_admin_credentials so a value resolved interactively
457
+ # is what actually gets checked, not whatever (possibly empty) value
458
+ # cfg.app.admin_password held before resolution.
459
+ if cfg.manual.participant_exposure != "none":
460
+ from lablink_allocator_service.validate_config import is_weak_admin_password
461
+
462
+ if is_weak_admin_password(admin_pw):
463
+ console.print(
464
+ f"[red]manual.participant_exposure is "
465
+ f"'{cfg.manual.participant_exposure}' but the resolved admin "
466
+ "password is empty, a known example value, or shorter than "
467
+ "12 characters.[/red]\n"
468
+ "A publicly exposed allocator is reachable from the internet "
469
+ "and gets scanned within minutes — set a strong "
470
+ "admin_password (12+ characters, not a common default) "
471
+ "before deploying."
472
+ )
473
+ raise SystemExit(1)
474
+
475
+ if not yes:
476
+ action = "create" if not target.exists() else "update"
477
+ console.print(
478
+ f"About to {action} compose stack in {target}\n"
479
+ f" provider: manual\n"
480
+ f" ssl: {cfg.ssl.provider}\n"
481
+ f" admin user: {admin_user}\n"
482
+ )
483
+ if not typer.confirm("Proceed?", default=True):
484
+ console.print("Aborted.")
485
+ raise SystemExit(1)
486
+
487
+ # Initialize deployment metrics — written incrementally so a failed
488
+ # deploy still leaves a record on disk, same as the AWS path. Started
489
+ # here, after the confirmation gate, so an aborted "Proceed?" leaves no
490
+ # in_progress file behind at all. region/template_version stay None:
491
+ # there is no region and no OpenTofu template in a compose deploy.
492
+ deploy_start_dt = datetime.now(timezone.utc)
493
+ metrics = DeploymentMetrics(
494
+ deployment_name=cfg.deployment_name,
495
+ provider="manual",
496
+ ssl_enabled=cfg.ssl.provider != "none",
497
+ allocator_deploy_start_time=deploy_start_dt.isoformat(),
498
+ )
499
+ metrics_path = cache_path_for(cfg.deployment_name, deploy_start_dt)
500
+ write_metrics(metrics_path, metrics)
501
+
502
+ # Everything from here to the success write is inside the try: the record
503
+ # already exists on disk, so any escape that skips the write below strands
504
+ # it at in_progress — indistinguishable from a Ctrl-C, and with null
505
+ # timings. render_compose_dir writes files and calls save_config, and
506
+ # _write_canonical_url writes one too, so OSError is live in both stretches
507
+ # either side of the timed phases, not just in the phases themselves.
508
+ try:
509
+ render_compose_dir(
510
+ cfg,
511
+ target,
512
+ tailscale_authkey=tailscale_authkey,
513
+ cloudflare_tunnel_token=cloudflare_tunnel_token,
514
+ )
515
+ console.print(f"[green]Rendered {target}[/green]")
516
+
517
+ # Explicitly disable Funnel *before* _compose_up, whenever the new
518
+ # config no longer wants it — this must run before --remove-orphans
519
+ # potentially deletes the sidecar (a removed container can't be
520
+ # `docker exec`'d into), and it's needed even when the sidecar
521
+ # sticks around unchanged (e.g. connectivity=mesh_overlay alone),
522
+ # since Funnel persists in the sidecar's own state regardless of
523
+ # whether _enable_funnel keeps getting called. See _disable_funnel.
524
+ if cfg.manual.participant_exposure != "tailscale_funnel":
525
+ _disable_funnel(docker=docker)
526
+
527
+ with phase_timer(
528
+ metrics, "allocator_compose_up_duration_seconds", metrics_path
529
+ ):
530
+ _compose_up(target, docker=docker)
531
+ with phase_timer(
532
+ metrics, "allocator_health_check_duration_seconds", metrics_path
533
+ ):
534
+ _health_poll(docker=docker)
535
+
536
+ # Disable again, now that the sidecar (if the compose file still
537
+ # declares one) is guaranteed running. The call above can silently
538
+ # no-op if the sidecar was stopped-but-not-removed at that point —
539
+ # `docker exec` fails on a stopped container the same way it does on
540
+ # a missing one, and _disable_funnel() can't tell those apart. If
541
+ # connectivity stays mesh_overlay, _compose_up just restarted that
542
+ # same stopped sidecar, reattached to tailscale_state with Funnel's
543
+ # last-known "on" config still intact — this second call is what
544
+ # actually clears it. Harmless no-op if the sidecar was removed as
545
+ # an orphan instead (nothing to disable).
546
+ if cfg.manual.participant_exposure != "tailscale_funnel":
547
+ _disable_funnel(docker=docker)
548
+
549
+ funnel_ok = True
550
+ funnel_url = None
551
+ if cfg.manual.participant_exposure == "tailscale_funnel":
552
+ funnel_ok, funnel_url = _enable_funnel(docker=docker)
553
+ if funnel_url:
554
+ _write_canonical_url(target, funnel_url)
555
+
556
+ if cfg.manual.participant_exposure == "cloudflare_tunnel":
557
+ if not _verify_public_hostname(cfg.manual.public_hostname):
558
+ console.print(
559
+ f"[yellow]https://{cfg.manual.public_hostname} did not "
560
+ "answer.[/yellow]\n"
561
+ "The stack is up locally. Common causes: the DNS record is "
562
+ "still propagating (retry in a few minutes), or the "
563
+ "tunnel's public hostname in Cloudflare does not point at "
564
+ "http://localhost:5000."
565
+ )
566
+ _print_last_log_lines(docker=docker)
567
+ except (Exception, SystemExit) as e:
568
+ # SystemExit IS caught here, unlike the AWS path where it means "user
569
+ # cancelled" and in_progress is the honest state. Nothing in this
570
+ # stretch is a cancellation — _compose_up and _health_poll both raise
571
+ # SystemExit for a genuine failure (non-zero compose exit, health
572
+ # timeout), so leaving those as in_progress would under-report real
573
+ # failures. KeyboardInterrupt is a BaseException and still escapes.
574
+ metrics.status = "failed"
575
+ metrics.error = str(e)
576
+ write_metrics(metrics_path, metrics)
577
+ raise
578
+
579
+ funnel_active = cfg.manual.participant_exposure == "tailscale_funnel" and funnel_ok
580
+
581
+ # Total = sum of the timed phases, matching the AWS path's definition
582
+ # (machine work only, excluding prompt time). Exposure setup — Funnel
583
+ # enable, public-hostname verification — is deliberately untimed, so a
584
+ # slow DNS propagation doesn't masquerade as slow deploy machinery.
585
+ metrics.allocator_deploy_end_time = datetime.now(timezone.utc).isoformat()
586
+ metrics.allocator_total_deployment_duration_seconds = round(
587
+ sum(
588
+ v
589
+ for v in (
590
+ metrics.allocator_compose_up_duration_seconds,
591
+ metrics.allocator_health_check_duration_seconds,
592
+ )
593
+ if v is not None
594
+ ),
595
+ 3,
596
+ )
597
+ metrics.status = "success" if funnel_ok else "failed"
598
+ if not funnel_ok:
599
+ metrics.error = "tailscale funnel could not be enabled"
600
+ write_metrics(metrics_path, metrics)
601
+
602
+ _print_summary(
603
+ cfg, funnel_active=funnel_active, funnel_url=funnel_url, docker=docker
604
+ )
605
+
606
+ if not funnel_ok:
607
+ raise SystemExit(1)
608
+
609
+
610
+ def _verify_public_hostname(hostname: str) -> bool:
611
+ """True if the allocator answers on its public hostname.
612
+
613
+ One request over the real public URL proves DNS resolution, the
614
+ Cloudflare edge, the tunnel, nginx and Flask together — checking that
615
+ cloudflared is alive says nothing about whether the edge found it.
616
+
617
+ Polled, because the local `_health_poll` above clears as soon as Flask
618
+ answers — which is *before* the public path exists. cloudflared is still
619
+ registering its edge connections at that point and nginx is still binding
620
+ :5000, so a single attempt fails on a perfectly good deploy (observed
621
+ live 2026-08-05: warned at 20:25:30, the same container served the public
622
+ hostname seconds later and stayed up).
623
+
624
+ Still advisory, and bounded short: the remaining failure modes — a
625
+ still-propagating DNS record, or a public hostname in the Cloudflare
626
+ dashboard pointing somewhere other than the origin — are not things
627
+ waiting fixes, and the caller only warns.
628
+ """
629
+ url = f"https://{hostname}"
630
+ console.print(
631
+ f"[bold]Verifying public hostname {url}/api/health "
632
+ f"(up to {PUBLIC_HOSTNAME_MAX_ATTEMPTS} tries) …[/bold]"
633
+ )
634
+ for attempt in range(1, PUBLIC_HOSTNAME_MAX_ATTEMPTS + 1):
635
+ try:
636
+ healthy = bool(check_health_endpoint(url).get("healthy"))
637
+ except OSError:
638
+ # Unresolvable name / refused connection while the record
639
+ # propagates, or while the tunnel route is still coming up.
640
+ healthy = False
641
+ if healthy:
642
+ console.print(f"[green]Public hostname is live: {url}[/green]")
643
+ return True
644
+ if attempt < PUBLIC_HOSTNAME_MAX_ATTEMPTS:
645
+ time.sleep(PUBLIC_HOSTNAME_RETRY_DELAY_SECONDS)
646
+ return False
647
+
648
+
649
+ def _compose_up(target: Path, *, docker: Docker) -> None:
650
+ console.print("[bold]docker compose up -d …[/bold]")
651
+ # --remove-orphans: if needs_sidecar just became False (connectivity
652
+ # switched off mesh_overlay AND participant_exposure switched off
653
+ # tailscale_funnel), the freshly-rendered compose file no longer
654
+ # declares the tailscale service — without this flag, `docker
655
+ # compose up` leaves that now-undeclared container running
656
+ # untouched, forever. _disable_funnel() (called before this, in
657
+ # run_deploy_compose) already clears its Funnel state first, so
658
+ # this just ensures the container itself doesn't linger too.
659
+ result = docker.compose(target, "up", "-d", "--remove-orphans", capture=False)
660
+ if not result.ok:
661
+ console.print("[red]docker compose up failed.[/red]")
662
+ raise SystemExit(result.returncode or 1)
663
+
664
+
665
+ def _write_canonical_url(target: Path, url: str) -> None:
666
+ """Publish the allocator's real public URL to the bind-mounted file the
667
+ allocator reads (see config_helpers.canonical_base_url).
668
+
669
+ Behind Funnel the allocator cannot work its own public URL out from the
670
+ request: Funnel injects no X-Forwarded-Proto, and manual deployments run
671
+ ssl.provider=none so the header-trust gate is shut anyway. It therefore
672
+ reports http://, which clients can only follow via a 302 that downgrades
673
+ their POSTs to GET. This file is the out-of-band channel that fixes that,
674
+ carrying the address `tailscale funnel status` actually reported — which
675
+ also picks up the numeric hostname suffixes (-2, -3, ...) that a name
676
+ collision with an offline node from a prior deploy produces.
677
+
678
+ Written IN PLACE, never via a temp file + rename: docker bind-mounts a
679
+ single file by inode, so a rename would leave the running container
680
+ reading the old file forever.
681
+ """
682
+ path = target / CANONICAL_URL_FILENAME
683
+ with path.open("w") as f:
684
+ f.write(f"{url.rstrip('/')}\n")
685
+
686
+
687
+ FUNNEL_STATUS_URL_RE = re.compile(r"(https://\S+)\s*\(Funnel on\)")
688
+
689
+
690
+ def _funnel_status_url(*, docker: Docker) -> str | None:
691
+ """Query the sidecar for the public URL Tailscale Funnel is actually
692
+ serving right now, via `tailscale funnel status`.
693
+
694
+ This is the authoritative source for the URL — Tailscale assigns the
695
+ node's hostname, and it does not necessarily match
696
+ `lablink-allocator-<deployment_name>`: a name collision with an
697
+ existing (possibly offline) tailnet node from a prior deploy gets a
698
+ numeric suffix appended instead (verified live: `-2`, `-3`, ... after
699
+ repeated deploy/destroy cycles). Returns None if Funnel isn't active
700
+ or the output didn't match the expected format.
701
+ """
702
+ result = docker.exec_in(
703
+ TAILSCALE_SIDECAR_CONTAINER_NAME,
704
+ ["tailscale", "funnel", "status"],
705
+ )
706
+ match = FUNNEL_STATUS_URL_RE.search(result.stdout)
707
+ return match.group(1) if match else None
708
+
709
+
710
+ def _enable_funnel(*, docker: Docker) -> tuple[bool, str | None]:
711
+ """Idempotently enable Tailscale Funnel on the allocator's own nginx
712
+ port, via the sidecar container.
713
+
714
+ `tailscale funnel`'s config lives in tailscaled's own local state
715
+ (already persisted by the compose file's `tailscale_state` named
716
+ volume), so this is safe to re-run on every deploy — a no-op if
717
+ already enabled. If the tailnet hasn't granted the Funnel ACL yet,
718
+ the command's own output names the exact grant URL; this surfaces
719
+ that URL and returns (False, None) rather than silently leaving the
720
+ allocator unreachable to participants.
721
+
722
+ Retries a few times with a short delay: the sidecar may still be
723
+ completing its own `tailscale up` join when this runs (right after
724
+ `_compose_up`/`_health_poll`, which only confirm the *allocator*
725
+ container is healthy, not the sidecar's tailnet membership) — the
726
+ same class of startup race already fixed on the client side (commit
727
+ 7a8ab9f6). Only retried for transient not-ready-yet failures; an
728
+ ACL-not-granted response is unambiguous and returned immediately
729
+ without retrying, since retrying can't fix a missing grant.
730
+
731
+ Returns (True, url) if Funnel is enabled (or already was) — url is
732
+ the real address from `_funnel_status_url()`, or None if that lookup
733
+ didn't find one despite the enable itself succeeding. Returns
734
+ (False, None) otherwise (ACL not granted, or failure persisting
735
+ across all retries) — callers should still let the rest of the
736
+ deploy complete either way (the stack is functional for LAN/
737
+ mesh-overlay access regardless), but should ultimately exit non-zero
738
+ when this returns False.
739
+ """
740
+ for attempt in range(1, FUNNEL_ENABLE_MAX_ATTEMPTS + 1):
741
+ result = docker.exec_in(
742
+ TAILSCALE_SIDECAR_CONTAINER_NAME,
743
+ ["tailscale", "funnel", "--bg", str(ALLOCATOR_INTERNAL_PORT)],
744
+ )
745
+ output = result.stdout + result.stderr
746
+ if FUNNEL_ACL_NOT_GRANTED_MARKER in output:
747
+ console.print(
748
+ "[yellow]Tailscale Funnel isn't authorized on this tailnet "
749
+ "yet.[/yellow] The compose stack is up and reachable on your "
750
+ "LAN, but participant exposure needs a one-time grant:\n"
751
+ )
752
+ console.print(output.strip())
753
+ return False, None
754
+ if result.returncode == 0:
755
+ console.print(
756
+ "[green]Tailscale Funnel enabled for participant access.[/green]"
757
+ )
758
+ console.print(output.strip())
759
+ return True, _funnel_status_url(docker=docker)
760
+ if attempt < FUNNEL_ENABLE_MAX_ATTEMPTS:
761
+ time.sleep(FUNNEL_ENABLE_RETRY_DELAY_SECONDS)
762
+ continue
763
+ console.print(
764
+ f"[red]Failed to enable Tailscale Funnel after "
765
+ f"{FUNNEL_ENABLE_MAX_ATTEMPTS} attempts (exit "
766
+ f"{result.returncode}):[/red]\n{output.strip()}"
767
+ )
768
+ return False, None
769
+
770
+
771
+ def _disable_funnel(*, docker: Docker) -> None:
772
+ """Explicitly clear Tailscale Funnel's serve config on the sidecar,
773
+ best-effort.
774
+
775
+ `tailscale funnel --bg` persists in tailscaled's own local state (the
776
+ `tailscale_state` named volume) across container restarts — and even
777
+ across container *recreation*, since a freshly-created sidecar
778
+ reattaches to that same volume and the same node identity resumes
779
+ serving from its last-known config. Simply no longer calling
780
+ `_enable_funnel()` is NOT enough to actually turn Funnel off; it has
781
+ to be explicitly disabled, or a previously-Funnel-exposed allocator
782
+ stays publicly reachable even after an operator sets
783
+ participant_exposure back to "none".
784
+
785
+ Called whenever the new config's participant_exposure is no longer
786
+ "tailscale_funnel", *before* `_compose_up` — including the case
787
+ where the sidecar is about to be removed entirely as a compose
788
+ orphan (needs_sidecar became False), since a removed container can
789
+ no longer be `docker exec`'d into and its persisted volume would
790
+ otherwise carry the stale "enabled" state forward to any future
791
+ sidecar that reattaches to it.
792
+
793
+ Best-effort and silent on failure: if the sidecar container doesn't
794
+ exist at all (e.g. a fresh deployment that never enabled Funnel),
795
+ there is nothing to disable and no error is surfaced.
796
+ """
797
+ result = docker.exec_in(
798
+ TAILSCALE_SIDECAR_CONTAINER_NAME,
799
+ ["tailscale", "funnel", "--https=443", "off"],
800
+ )
801
+ if result.ok:
802
+ console.print("[dim]Tailscale Funnel disabled.[/dim]")
803
+
804
+
805
+ def _health_poll(*, docker: Docker) -> None:
806
+ """Poll the allocator's /api/health on localhost until healthy."""
807
+ # Manual provider is HTTP-only; the host port comes from the rendered
808
+ # .env, which defaults to DEFAULT_HTTP_PORT.
809
+ base_url = f"http://localhost:{DEFAULT_HTTP_PORT}"
810
+
811
+ console.print(
812
+ f"[bold]Polling allocator health at {base_url}/api/health "
813
+ f"(up to {HEALTH_POLL_TIMEOUT_SECONDS}s) …[/bold]"
814
+ )
815
+ start = time.monotonic()
816
+ deadline = start + HEALTH_POLL_TIMEOUT_SECONDS
817
+ while time.monotonic() < deadline:
818
+ result = check_health_endpoint(base_url)
819
+ if result.get("healthy"):
820
+ elapsed = time.monotonic() - start
821
+ console.print(f"[green]Allocator healthy after {elapsed:.0f}s[/green]")
822
+ return
823
+ time.sleep(3)
824
+
825
+ console.print(
826
+ "[yellow]Allocator did not become healthy within "
827
+ f"{HEALTH_POLL_TIMEOUT_SECONDS}s.[/yellow]"
828
+ )
829
+ _print_last_log_lines(docker=docker)
830
+ raise SystemExit(1)
831
+
832
+
833
+ def _redact_secrets(text: str) -> str:
834
+ """Blank out credential values in container output before printing it.
835
+
836
+ Keyed on the variable *name*, not the value's shape: a Cloudflare token
837
+ is base64-ish and a tailnet auth key is `tskey-`-prefixed, but both are
838
+ the vendor's to change, whereas the names are ours.
839
+
840
+ Needed because `cloudflared` logs its whole environment at INFO on
841
+ startup. `start.sh` unsets the token before launching it, so a current
842
+ image never logs it — this is the second layer, covering images built
843
+ before that change and any other path that echoes a secret.
844
+ """
845
+ return re.sub(
846
+ r"((?:CLOUDFLARE_TUNNEL_TOKEN|TS_AUTHKEY)[=:]|--token[= ])\S+",
847
+ r"\1<redacted>",
848
+ text,
849
+ )
850
+
851
+
852
+ def _print_last_log_lines(lines: int = 30, *, docker: Docker) -> None:
853
+ # Merge stderr: the allocator's Python logging goes there (see
854
+ # _extract_register_token), so capturing the streams separately and
855
+ # printing only stdout hid the very tracebacks this dump exists to
856
+ # surface. Merging is also why the redaction above is load-bearing.
857
+ result = docker.logs(ALLOCATOR_CONTAINER_NAME, tail=lines, merge_stderr=True)
858
+ if result.stdout:
859
+ console.print("[dim]Last allocator log lines:[/dim]")
860
+ console.print(_redact_secrets(result.stdout))
861
+
862
+
863
+ def _print_summary(
864
+ cfg: Config,
865
+ *,
866
+ funnel_active: bool = False,
867
+ funnel_url: str | None = None,
868
+ docker: Docker,
869
+ ) -> None:
870
+ register_token = _extract_register_token(docker=docker)
871
+ # Manual provider is HTTP-only; preflight rejects anything else.
872
+ local_url = "http://localhost"
873
+ lan_ip = _detect_lan_ip()
874
+ lan_url = f"http://{lan_ip}" if lan_ip else None
875
+ # BYO clients run on different boxes, so the register command needs
876
+ # an address those boxes can route to — localhost is only useful for
877
+ # self-registration on the operator's host. Prefer the LAN URL when
878
+ # we could detect one.
879
+ register_url = lan_url or local_url
880
+
881
+ # The deployment's one internet-reachable URL, or None. Funnel's has to
882
+ # be read back out of the sidecar and can be missing even when enabled;
883
+ # Cloudflare's is just config the admin typed. Everything downstream only
884
+ # cares whether such a URL exists, so both collapse into one value here
885
+ # rather than each exposure mode growing its own branch.
886
+ if funnel_active and funnel_url:
887
+ public_url = funnel_url
888
+ elif cfg.manual.participant_exposure == "cloudflare_tunnel":
889
+ public_url = f"https://{cfg.manual.public_hostname}"
890
+ else:
891
+ public_url = None
892
+
893
+ console.print("\n[bold green]Deployment complete.[/bold green]")
894
+ if funnel_active and not funnel_url:
895
+ console.print(
896
+ " Allocator URL (public): (enabled, but the URL could not be "
897
+ f"determined — run `docker exec {TAILSCALE_SIDECAR_CONTAINER_NAME} "
898
+ "tailscale funnel status` to see it)",
899
+ soft_wrap=True,
900
+ highlight=False,
901
+ )
902
+ elif public_url:
903
+ console.print(
904
+ f" Allocator URL (public): {public_url}",
905
+ soft_wrap=True,
906
+ highlight=False,
907
+ )
908
+ console.print(f" Allocator URL (local): {local_url}")
909
+ if lan_url:
910
+ console.print(f" Allocator URL (LAN): {lan_url}")
911
+ else:
912
+ # Be loud about *why* we couldn't pin a LAN address — operators
913
+ # who are routing through Tailscale/VPN/etc. need to know they
914
+ # have to substitute the right hostname themselves.
915
+ console.print(
916
+ " Allocator URL (LAN): (no LAN IP detected — pass the "
917
+ "operator host's reachable address manually)"
918
+ )
919
+ # public_url and lan_direct are mutually exclusive (preflight above), so
920
+ # this chain encodes the connectivity rule without re-reading it.
921
+ admin_url = public_url or lan_url or local_url
922
+ # soft_wrap: a real Funnel URL plus the column prefix overruns 80 cols.
923
+ console.print(
924
+ f" Admin URL: {admin_url}/admin",
925
+ soft_wrap=True,
926
+ highlight=False,
927
+ )
928
+ console.print(f" Admin user: {cfg.app.admin_user}")
929
+ if register_token:
930
+ console.print(f" Register token: {register_token}")
931
+ else:
932
+ # The allocator logs to stderr (Python `logging` default), so
933
+ # the recovery command MUST redirect stderr (`2>&1`) before the
934
+ # pipe — otherwise grep sees only the container's stdout and
935
+ # the user gets an empty result, same root cause as the bug this
936
+ # path is recovering from.
937
+ # soft_wrap=True keeps the docker-logs hint on a single line so
938
+ # the suggested command is not split mid-pipe in narrow terminals.
939
+ console.print(
940
+ " Register token: (could not parse from container "
941
+ "logs; fetch with `docker logs lablink-allocator 2>&1 | "
942
+ "grep REGISTER_TOKEN`)",
943
+ soft_wrap=True,
944
+ highlight=False,
945
+ )
946
+
947
+ # Print a copy-paste-ready command using the LAN URL when available
948
+ # (clients registering over the LAN can't reach localhost). The
949
+ # token-bearing line uses soft_wrap=True so narrow terminals don't
950
+ # insert a hard newline mid-command — that would break the
951
+ # operator's copy-paste.
952
+ mesh_overlay = cfg.manual.connectivity == "mesh_overlay"
953
+ reverse_tunnel = cfg.manual.connectivity == "reverse_tunnel"
954
+ # Both connectivity modes below mean the client isn't reachable on the
955
+ # allocator's own LAN — mesh_overlay via a Tailscale tailnet,
956
+ # reverse_tunnel by dialing out instead of accepting inbound at all.
957
+ off_lan = mesh_overlay or reverse_tunnel
958
+ # Substitute only when a real public URL exists — funnel_active can be
959
+ # True while funnel_url is None (enable succeeded but the status lookup
960
+ # didn't match), and a guessed fallback here would be exactly the wrong
961
+ # URL this function used to print. Gated on off_lan, not just
962
+ # mesh_overlay: a reverse_tunnel client behind a NAT'd/firewalled box is
963
+ # just as unreachable at the LAN address. Gated on public_url rather
964
+ # than on Funnel specifically, because an off-LAN client cannot reach
965
+ # the LAN address no matter which exposure mode publishes the
966
+ # allocator — printing one is how this hint was wrong before.
967
+ public_url_used = off_lan and bool(public_url)
968
+ # Hoisted above the mesh_overlay/reverse_tunnel branch so both off-LAN
969
+ # modes get the substitution — lan_direct clients genuinely are on the
970
+ # LAN, so their own hint below keeps using register_url as-is.
971
+ if public_url_used:
972
+ register_url = public_url
973
+ if mesh_overlay:
974
+ # A mesh-overlay client (e.g. a Run:AI-hosted workload) isn't on
975
+ # the allocator's LAN at all — the LAN URL above is unreachable
976
+ # from it regardless of whether we detected one. Whichever exposure
977
+ # mode is live, its public URL IS reachable from anywhere with
978
+ # internet access, so prefer that here.
979
+ console.print(
980
+ "\n[bold]Next step:[/bold] for each mesh-overlay client "
981
+ "(e.g. a Run:AI-hosted workload), open a terminal inside "
982
+ "that workload and run (hostname/machine-identity/GPU are "
983
+ "auto-detected):"
984
+ )
985
+ register_cmd = (
986
+ f" lablink client register --allocator-url {register_url} "
987
+ f"--register-token {register_token or '<token>'} "
988
+ "--overlay-hostname <name> --tailscale-authkey <key>"
989
+ )
990
+ elif reverse_tunnel:
991
+ console.print(
992
+ "\n[bold]Next step:[/bold] for each tunnel client (a box or "
993
+ "workload that can't accept inbound connections), open a "
994
+ "terminal inside it and run (hostname/machine-identity/GPU are "
995
+ "auto-detected; the tunnel's values are minted by the "
996
+ "allocator, so --tunnel takes no arguments):"
997
+ )
998
+ register_cmd = (
999
+ f" lablink client register --allocator-url {register_url} "
1000
+ f"--register-token {register_token or '<token>'} --tunnel"
1001
+ )
1002
+ else:
1003
+ console.print("\n[bold]Next step:[/bold] on each BYO box on the same LAN, run")
1004
+ register_cmd = (
1005
+ f" lablink client register --allocator-url {register_url} "
1006
+ f"--register-token {register_token or '<token>'}"
1007
+ )
1008
+ console.print(register_cmd, soft_wrap=True, highlight=False)
1009
+ if off_lan:
1010
+ console.print(
1011
+ " [dim]Registering ahead of time from elsewhere instead? "
1012
+ "Add --no-run-locally to print secrets for your own "
1013
+ "workload submission instead of running here, along with "
1014
+ "--hostname/--machine-identity.[/dim]"
1015
+ )
1016
+ if not lan_url and not public_url_used:
1017
+ # If we fell back to localhost, the printed command only works
1018
+ # for a BYO client *on the operator host*. Call that out so the
1019
+ # operator doesn't blindly hand it to a remote teammate. Doesn't
1020
+ # apply when the off-LAN hint above already substituted a public
1021
+ # URL instead of falling back to localhost.
1022
+ console.print(
1023
+ " [yellow]Note:[/yellow] the URL above is localhost — only "
1024
+ "valid for a BYO client running on this same machine. For "
1025
+ "clients on another box, substitute this host's LAN IP / "
1026
+ "hostname.",
1027
+ soft_wrap=True,
1028
+ highlight=False,
1029
+ )
1030
+
1031
+
1032
+ def _detect_lan_ip() -> str | None:
1033
+ """Best-effort: the IPv4 address another host on the operator's LAN
1034
+ would use to reach this machine. Returns ``None`` if we can't pick
1035
+ one (no default route, only loopback configured, …).
1036
+
1037
+ Uses the kernel routing-table trick: open a UDP socket and call
1038
+ ``connect()`` to a public IP. No packets are sent (UDP is
1039
+ connectionless), but the kernel resolves the route and binds the
1040
+ socket's local address — which we then read back via
1041
+ ``getsockname()``. Works offline as long as a default route exists.
1042
+ """
1043
+ s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
1044
+ try:
1045
+ # 8.8.8.8 is a well-known recipe target — we only need the
1046
+ # kernel to pick *an* outbound interface, nothing is transmitted.
1047
+ s.connect(("8.8.8.8", 80))
1048
+ ip = s.getsockname()[0]
1049
+ except OSError:
1050
+ return None
1051
+ finally:
1052
+ s.close()
1053
+
1054
+ # A loopback or unspecified address means the operator's box doesn't
1055
+ # have a usable LAN interface; treat that as "no LAN IP".
1056
+ if not ip or ip.startswith("127.") or ip == "0.0.0.0":
1057
+ return None
1058
+ return ip
1059
+
1060
+
1061
+ def _extract_register_token(*, docker: Docker) -> str | None:
1062
+ """Parse the register_token from the allocator's startup logs.
1063
+
1064
+ The allocator logs `REGISTER_TOKEN=<token>` at startup (grep for
1065
+ `REGISTER_TOKEN=%s` in `lablink_allocator_service/main.py`).
1066
+ Also tolerate the `register_token = "..."` form just in case.
1067
+
1068
+ Python's `logging.basicConfig` writes to stderr, so `merge_stderr=True`
1069
+ is required here too — see `Docker.logs`.
1070
+ """
1071
+ result = docker.logs(ALLOCATOR_CONTAINER_NAME, merge_stderr=True)
1072
+ if not result.ok:
1073
+ return None
1074
+ for pattern in (
1075
+ r'REGISTER_TOKEN\s*=\s*"?([A-Za-z0-9_\-]{20,})"?',
1076
+ r'register_token\s*=\s*"?([A-Za-z0-9_\-]{20,})"?',
1077
+ ):
1078
+ m = re.search(pattern, result.stdout)
1079
+ if m:
1080
+ return m.group(1)
1081
+ return None
1082
+
1083
+
1084
+ def _pgdata_volume_name(target: Path, *, docker: Docker) -> str | None:
1085
+ """Resolve the Docker volume currently backing the allocator's Postgres data.
1086
+
1087
+ Tries the running container's actual mount first (exact, no guessing).
1088
+ Falls back to Compose's own directory-basename project-naming
1089
+ convention when the container's already gone — e.g. an operator ran a
1090
+ manual `docker compose down` (removing containers, leaving volumes)
1091
+ before `lablink destroy` — verified via `docker volume inspect` before
1092
+ trusting it, so a wrong guess can't be silently mistaken for "nothing
1093
+ to remove". Guessing is safe here specifically because deployment_name
1094
+ (and therefore target.name) is already regex-constrained to Compose's
1095
+ own project-name character set (`^[a-z][a-z0-9-]*[a-z0-9]$` — see
1096
+ config/schema.py's DEPLOYMENT_NAME_RE), so there's no normalization
1097
+ mismatch to worry about.
1098
+
1099
+ Returns None only if no volume can be found by either method — i.e.
1100
+ this deployment never actually created one.
1101
+ """
1102
+ name = docker.inspect_format(
1103
+ ALLOCATOR_CONTAINER_NAME,
1104
+ '{{range .Mounts}}{{if eq .Destination "/var/lib/postgresql"}}'
1105
+ "{{.Name}}{{end}}{{end}}",
1106
+ )
1107
+ if name:
1108
+ return name
1109
+
1110
+ candidate = f"{target.name}_allocator_pgdata"
1111
+ return candidate if docker.volume_exists(candidate) else None
1112
+
1113
+
1114
+ def run_destroy_compose(
1115
+ cfg: Config,
1116
+ *,
1117
+ yes: bool = False,
1118
+ keep_data: bool = False,
1119
+ workdir_root: Path | None = None,
1120
+ docker: Docker | None = None,
1121
+ ) -> None:
1122
+ """Tear down a manual-provider compose stack.
1123
+
1124
+ Default behavior: wipes the Postgres data volume (all registration
1125
+ history, sessions, etc.) plus the working directory. A subsequent
1126
+ `lablink deploy` with the same deployment_name then starts from a
1127
+ genuinely empty database, matching what "destroy" means for every
1128
+ other provider — previously the default silently preserved the old
1129
+ volume, so a "fresh" redeploy kept showing every client registered
1130
+ under a prior deployment.
1131
+
1132
+ The Postgres volume is removed by name (resolved via `_pgdata_volume_name`)
1133
+ rather than via `docker compose down --volumes`, which would also delete
1134
+ the mesh-overlay `tailscale_state` volume — that volume carries the
1135
+ Tailscale node's identity, not "data": wiping it forces a brand-new
1136
+ tailnet registration on the next deploy, which changes the node's
1137
+ hostname (and any Funnel URL already handed to participants) for no
1138
+ reason. `tailscale_state` is always preserved, independent of `keep_data`.
1139
+
1140
+ With `keep_data=True`: no volumes are touched at all, and the working
1141
+ directory is left in place — re-deploying with the same deployment_name
1142
+ restores the previous DB state instead of starting fresh. Opt into this
1143
+ only if that's specifically what you want (e.g. a deliberate maintenance
1144
+ restart, not a real teardown).
1145
+
1146
+ `yes=True` skips the interactive confirmation prompt.
1147
+ `workdir_root` overrides `DEFAULT_COMPOSE_DIR` (used by tests).
1148
+ """
1149
+ docker = docker or default_docker()
1150
+ target = compose_workdir(cfg, workdir_root)
1151
+
1152
+ if not target.exists():
1153
+ console.print(
1154
+ f"[yellow]No compose stack at {target} — already destroyed.[/yellow]"
1155
+ )
1156
+ return
1157
+
1158
+ if not yes:
1159
+ if not keep_data:
1160
+ console.print(
1161
+ "[red bold]This will DELETE the Postgres data volume "
1162
+ "(all registration history, sessions, etc.). Pass "
1163
+ "--keep-data to preserve it instead.[/red bold]"
1164
+ )
1165
+ confirmation = typer.prompt(
1166
+ f"Type 'yes' to tear down compose stack at {target}",
1167
+ default="no",
1168
+ show_default=False,
1169
+ )
1170
+ if confirmation.strip().lower() != "yes":
1171
+ console.print("Aborted.")
1172
+ raise SystemExit(1)
1173
+
1174
+ pgdata_volume = None if keep_data else _pgdata_volume_name(target, docker=docker)
1175
+
1176
+ result = docker.compose(target, "down", capture=False)
1177
+ if not result.ok:
1178
+ console.print("[red]docker compose down failed.[/red]")
1179
+ raise SystemExit(result.returncode or 1)
1180
+
1181
+ if not keep_data:
1182
+ if pgdata_volume:
1183
+ rm_result = docker.remove_volume(pgdata_volume)
1184
+ if not rm_result.ok:
1185
+ console.print(
1186
+ f"[red]Failed to remove Postgres volume "
1187
+ f"{pgdata_volume}:[/red] {rm_result.stderr.strip()}\n"
1188
+ "The working directory was NOT removed — a later "
1189
+ "deploy could otherwise silently reattach to this "
1190
+ "volume's old data. Resolve the error above and "
1191
+ "re-run `lablink destroy`."
1192
+ )
1193
+ raise SystemExit(1)
1194
+ shutil.rmtree(target)
1195
+ console.print(f"[green]Removed {target}.[/green]")
1196
+ else:
1197
+ console.print(f"[green]Stack torn down (data preserved in {target}).[/green]")
1198
+
1199
+ console.print(
1200
+ "\n[bold]Reminder:[/bold] each BYO client box still has "
1201
+ "`lablink-client` running.\n"
1202
+ "Run [bold]lablink client unregister[/bold] on each box to clean up."
1203
+ )