lablink-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lablink_cli/__init__.py +8 -0
- lablink_cli/api.py +428 -0
- lablink_cli/app.py +938 -0
- lablink_cli/byo_detect.py +112 -0
- lablink_cli/commands/__init__.py +0 -0
- lablink_cli/commands/cleanup.py +647 -0
- lablink_cli/commands/deploy.py +863 -0
- lablink_cli/commands/deploy_compose.py +1203 -0
- lablink_cli/commands/doctor.py +549 -0
- lablink_cli/commands/export_metrics.py +244 -0
- lablink_cli/commands/launch.py +236 -0
- lablink_cli/commands/logs.py +434 -0
- lablink_cli/commands/register.py +839 -0
- lablink_cli/commands/reset_overlay.py +109 -0
- lablink_cli/commands/setup.py +347 -0
- lablink_cli/commands/stats.py +133 -0
- lablink_cli/commands/status.py +934 -0
- lablink_cli/commands/unregister.py +188 -0
- lablink_cli/commands/utils.py +552 -0
- lablink_cli/config/__init__.py +0 -0
- lablink_cli/config/schema.py +212 -0
- lablink_cli/deployment_metrics.py +94 -0
- lablink_cli/docker.py +419 -0
- lablink_cli/log_shipper.py +441 -0
- lablink_cli/templates/docker-compose.tailscale-override.yml +55 -0
- lablink_cli/templates/docker-compose.yml +67 -0
- lablink_cli/tofu_source.py +169 -0
- lablink_cli/tui/__init__.py +0 -0
- lablink_cli/tui/logs_viewer.py +413 -0
- lablink_cli/tui/wizard.py +1814 -0
- lablink_cli-0.1.0.dist-info/METADATA +76 -0
- lablink_cli-0.1.0.dist-info/RECORD +35 -0
- lablink_cli-0.1.0.dist-info/WHEEL +5 -0
- lablink_cli-0.1.0.dist-info/entry_points.txt +2 -0
- lablink_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1203 @@
|
|
|
1
|
+
"""`lablink deploy/destroy` — manual-provider compose orchestration.
|
|
2
|
+
|
|
3
|
+
The allocator image is monolithic: it bundles Flask + nginx + an internal
|
|
4
|
+
Postgres. This module renders a single-service docker-compose stack
|
|
5
|
+
(plus a `.env` and `config.yaml`) into a per-deployment workdir under
|
|
6
|
+
`~/.lablink/compose/<deployment_name>/`, runs `docker compose up -d`,
|
|
7
|
+
polls the allocator's `/api/health` endpoint, then prints a summary
|
|
8
|
+
including the register-token that BYO clients use to join.
|
|
9
|
+
|
|
10
|
+
Admin/DB credentials live inside the rendered `config.yaml` (mounted at
|
|
11
|
+
`/config/config.yaml`), not in env vars — the allocator container does
|
|
12
|
+
not read those from the environment.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
import shutil
|
|
19
|
+
import socket
|
|
20
|
+
import time
|
|
21
|
+
from datetime import datetime, timezone
|
|
22
|
+
from importlib import resources
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
import typer
|
|
26
|
+
from rich.console import Console
|
|
27
|
+
|
|
28
|
+
from lablink_cli.commands.status import check_health_endpoint
|
|
29
|
+
from lablink_cli.commands.utils import resolve_admin_credentials
|
|
30
|
+
from lablink_cli.config.schema import Config, save_config
|
|
31
|
+
from lablink_cli.deployment_metrics import (
|
|
32
|
+
DeploymentMetrics,
|
|
33
|
+
cache_path_for,
|
|
34
|
+
phase_timer,
|
|
35
|
+
write_metrics,
|
|
36
|
+
)
|
|
37
|
+
from lablink_cli.docker import Docker, DockerUnavailable, default_docker
|
|
38
|
+
|
|
39
|
+
DEFAULT_COMPOSE_DIR = Path.home() / ".lablink" / "compose"
|
|
40
|
+
DEFAULT_HTTP_PORT = "80"
|
|
41
|
+
HEALTH_POLL_TIMEOUT_SECONDS = 300
|
|
42
|
+
ALLOCATOR_IMAGE_BASE = "ghcr.io/talmolab/lablink-allocator-image"
|
|
43
|
+
# Only ssl=none is supported by the manual-provider compose stack today:
|
|
44
|
+
# the allocator image has no TLS terminator (Caddy is part of the AWS
|
|
45
|
+
# infrastructure, not the container). For public TLS, operators front the
|
|
46
|
+
# stack with their own reverse proxy.
|
|
47
|
+
SUPPORTED_SSL_FOR_MANUAL = ("none",)
|
|
48
|
+
ALLOCATOR_CONTAINER_NAME = "lablink-allocator"
|
|
49
|
+
TAILSCALE_SIDECAR_CONTAINER_NAME = "lablink-allocator-tailscale"
|
|
50
|
+
# The allocator's own nginx port inside the sidecar's shared network
|
|
51
|
+
# namespace — same target the manual `tailscale funnel 5000` spike used.
|
|
52
|
+
ALLOCATOR_INTERNAL_PORT = 5000
|
|
53
|
+
# Exact substring from `tailscale funnel`'s own output when the tailnet
|
|
54
|
+
# hasn't granted the Funnel ACL yet (verified live, 2026-07-22 spike).
|
|
55
|
+
FUNNEL_ACL_NOT_GRANTED_MARKER = "Funnel is not enabled on your tailnet"
|
|
56
|
+
FUNNEL_ENABLE_MAX_ATTEMPTS = 5
|
|
57
|
+
FUNNEL_ENABLE_RETRY_DELAY_SECONDS = 2
|
|
58
|
+
# Budget for the public-hostname check to survive cloudflared's own startup
|
|
59
|
+
# (see _verify_public_hostname). Live runs registered the first edge
|
|
60
|
+
# connection ~1s after `docker compose up` returned and served traffic within
|
|
61
|
+
# a few seconds; 6 tries 5s apart leaves ~25s of headroom over that.
|
|
62
|
+
PUBLIC_HOSTNAME_MAX_ATTEMPTS = 6
|
|
63
|
+
PUBLIC_HOSTNAME_RETRY_DELAY_SECONDS = 5
|
|
64
|
+
# Name of the file carrying the allocator's real public URL, staged next to
|
|
65
|
+
# config.yaml and bind-mounted to /config/<name>. Must stay in sync with
|
|
66
|
+
# config_helpers.CANONICAL_URL_FILENAME in the allocator package — duplicated
|
|
67
|
+
# rather than imported because each package's CI job installs only its own
|
|
68
|
+
# dependencies, so a cross-package import would fail there. Guarded by
|
|
69
|
+
# test_deploy_compose.py::TestCanonicalUrlFile::test_filename_matches_allocator.
|
|
70
|
+
CANONICAL_URL_FILENAME = "allocator-url"
|
|
71
|
+
|
|
72
|
+
console = Console()
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def compose_workdir(cfg: Config, root: Path | None = None) -> Path:
|
|
76
|
+
"""Path to the rendered compose working directory for this deployment.
|
|
77
|
+
|
|
78
|
+
`root` overrides `DEFAULT_COMPOSE_DIR` (used by tests via `workdir_root`).
|
|
79
|
+
"""
|
|
80
|
+
name = cfg.deployment_name or "lablink"
|
|
81
|
+
return (root or DEFAULT_COMPOSE_DIR) / name
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _read_env_value(env_path: Path, key: str) -> str | None:
|
|
85
|
+
"""Read a single KEY=value line from an existing .env file.
|
|
86
|
+
|
|
87
|
+
Used to carry TS_AUTHKEY forward across redeploys without requiring
|
|
88
|
+
the admin to re-supply --tailscale-authkey every time — tailscaled's
|
|
89
|
+
own state (the tailscale_state volume) is what actually matters after
|
|
90
|
+
the first join, but the sidecar's compose environment still needs
|
|
91
|
+
*some* value on every render.
|
|
92
|
+
"""
|
|
93
|
+
if not env_path.exists():
|
|
94
|
+
return None
|
|
95
|
+
prefix = f"{key}="
|
|
96
|
+
for line in env_path.read_text().splitlines():
|
|
97
|
+
if line.startswith(prefix):
|
|
98
|
+
return line[len(prefix) :]
|
|
99
|
+
return None
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _needs_tailscale_sidecar(cfg: Config) -> bool:
|
|
103
|
+
"""True if a tailnet join is needed for either of two independent
|
|
104
|
+
reasons: reaching mesh-overlay clients, or publishing the allocator
|
|
105
|
+
itself to participants via Funnel. Both reuse the same sidecar."""
|
|
106
|
+
return (
|
|
107
|
+
cfg.manual.connectivity == "mesh_overlay"
|
|
108
|
+
or cfg.manual.participant_exposure == "tailscale_funnel"
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _tailscale_state_volume_exists(target: Path, *, docker: Docker) -> bool:
|
|
113
|
+
"""True if this deployment's `tailscale_state` volume already exists.
|
|
114
|
+
|
|
115
|
+
Default `lablink destroy` preserves this volume (it carries the
|
|
116
|
+
sidecar's Tailscale node identity, not "data") but removes the whole
|
|
117
|
+
working directory, including the `.env` that would otherwise carry
|
|
118
|
+
TS_AUTHKEY forward. Without this check, a redeploy after such a
|
|
119
|
+
destroy would demand a fresh --tailscale-authkey purely because
|
|
120
|
+
there's no .env to read one from — even though the sidecar's identity
|
|
121
|
+
is already authenticated and sitting in this preserved volume, and
|
|
122
|
+
containerboot skips the `tailscale up --authkey` step when valid
|
|
123
|
+
state is already present. Guessed the same way as
|
|
124
|
+
`_pgdata_volume_name`'s fallback (verified via `docker volume
|
|
125
|
+
inspect`, safe because target.name is regex-constrained to Compose's
|
|
126
|
+
own project-name character set).
|
|
127
|
+
"""
|
|
128
|
+
return docker.volume_exists(f"{target.name}_tailscale_state")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def render_compose_dir(
|
|
132
|
+
cfg: Config,
|
|
133
|
+
target: Path,
|
|
134
|
+
*,
|
|
135
|
+
tailscale_authkey: str | None = None,
|
|
136
|
+
cloudflare_tunnel_token: str | None = None,
|
|
137
|
+
) -> None:
|
|
138
|
+
"""Render docker-compose.yml + .env + config.yaml into target.
|
|
139
|
+
|
|
140
|
+
The allocator image is monolithic (bundles its own Postgres), so the
|
|
141
|
+
compose stack is single-service — plus a `tailscale` sidecar service
|
|
142
|
+
whenever a tailnet join is needed for either of two independent
|
|
143
|
+
reasons: `cfg.manual.connectivity == "mesh_overlay"` (network_mode:
|
|
144
|
+
service:allocator, so the allocator's own nginx can route to a
|
|
145
|
+
mesh-overlay client's Tailscale hostname) or
|
|
146
|
+
`cfg.manual.participant_exposure == "tailscale_funnel"` (so the
|
|
147
|
+
allocator can publish itself to participants via Funnel). Both reuse
|
|
148
|
+
the exact same sidecar — it doesn't care which reason applies, which
|
|
149
|
+
is why it ships as one `docker-compose.override.yml` layered over the
|
|
150
|
+
single base stack rather than as a second full copy of it. The
|
|
151
|
+
internal Postgres data is persisted via a named volume on
|
|
152
|
+
/var/lib/postgresql. Admin/DB creds live in the saved config.yaml
|
|
153
|
+
(NOT in env vars) — the caller is responsible for populating
|
|
154
|
+
cfg.app.admin_user/admin_password (via `resolve_admin_credentials`)
|
|
155
|
+
before invoking this helper.
|
|
156
|
+
|
|
157
|
+
`tailscale_authkey` is only meaningful when the sidecar is needed. It
|
|
158
|
+
is not persisted in config.yaml (unlike admin/DB creds) — only into
|
|
159
|
+
this deployment's .env, and only for as long as the sidecar needs it
|
|
160
|
+
to join for the first time.
|
|
161
|
+
|
|
162
|
+
`cloudflare_tunnel_token` is only meaningful when
|
|
163
|
+
`cfg.manual.participant_exposure == "cloudflare_tunnel"`, and follows
|
|
164
|
+
the same rules as `tailscale_authkey`: .env only, carried forward on a
|
|
165
|
+
redeploy that omits it, overridden when supplied again (rotation).
|
|
166
|
+
"""
|
|
167
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
168
|
+
needs_sidecar = _needs_tailscale_sidecar(cfg)
|
|
169
|
+
|
|
170
|
+
# 1. Copy the bundled compose templates. The base stack is always the
|
|
171
|
+
# same file; the Tailscale sidecar arrives as Compose's own
|
|
172
|
+
# auto-loaded `docker-compose.override.yml`, so no `docker compose`
|
|
173
|
+
# call site needs `-f` flags. Delete a stale override when the
|
|
174
|
+
# sidecar is no longer needed — Compose would otherwise keep
|
|
175
|
+
# merging it and silently rejoin the tailnet on the next redeploy.
|
|
176
|
+
templates = resources.files("lablink_cli.templates")
|
|
177
|
+
(target / "docker-compose.yml").write_text(
|
|
178
|
+
templates.joinpath("docker-compose.yml").read_text()
|
|
179
|
+
)
|
|
180
|
+
override_path = target / "docker-compose.override.yml"
|
|
181
|
+
if needs_sidecar:
|
|
182
|
+
override_path.write_text(
|
|
183
|
+
templates.joinpath("docker-compose.tailscale-override.yml").read_text()
|
|
184
|
+
)
|
|
185
|
+
else:
|
|
186
|
+
override_path.unlink(missing_ok=True)
|
|
187
|
+
|
|
188
|
+
# 2. Render .env — only the values the compose template substitutes.
|
|
189
|
+
# No DB or admin creds here: they're inside config.yaml. Read the
|
|
190
|
+
# OLD .env (if any) before overwriting it, so a redeploy that
|
|
191
|
+
# omits --tailscale-authkey carries the previous value forward
|
|
192
|
+
# instead of blanking out an already-joined sidecar's key.
|
|
193
|
+
env_path = target / ".env"
|
|
194
|
+
previous_authkey = _read_env_value(env_path, "TS_AUTHKEY")
|
|
195
|
+
previous_cf_token = _read_env_value(env_path, "CLOUDFLARE_TUNNEL_TOKEN")
|
|
196
|
+
|
|
197
|
+
allocator_image = _allocator_image(cfg)
|
|
198
|
+
env_lines = [
|
|
199
|
+
f"ALLOCATOR_IMAGE={allocator_image}",
|
|
200
|
+
f"HTTP_PORT={DEFAULT_HTTP_PORT}",
|
|
201
|
+
# Always declared: the compose templates substitute it
|
|
202
|
+
# unconditionally, and an unset variable makes `docker compose up`
|
|
203
|
+
# warn on every deploy.
|
|
204
|
+
f"PARTICIPANT_EXPOSURE={cfg.manual.participant_exposure}",
|
|
205
|
+
]
|
|
206
|
+
if cfg.manual.participant_exposure == "cloudflare_tunnel":
|
|
207
|
+
# Same carry-forward rule as TS_AUTHKEY: a redeploy that omits the
|
|
208
|
+
# flag keeps the working token, while an explicitly supplied one
|
|
209
|
+
# wins (that is the rotation path).
|
|
210
|
+
env_lines.append(
|
|
211
|
+
f"CLOUDFLARE_TUNNEL_TOKEN="
|
|
212
|
+
f"{cloudflare_tunnel_token or previous_cf_token or ''}"
|
|
213
|
+
)
|
|
214
|
+
if needs_sidecar:
|
|
215
|
+
resolved_authkey = tailscale_authkey or previous_authkey or ""
|
|
216
|
+
env_lines.append(f"TS_AUTHKEY={resolved_authkey}")
|
|
217
|
+
env_lines.append(
|
|
218
|
+
f"TAILSCALE_HOSTNAME=lablink-allocator-{cfg.deployment_name or 'lablink'}"
|
|
219
|
+
)
|
|
220
|
+
env_path.write_text("\n".join(env_lines) + "\n")
|
|
221
|
+
env_path.chmod(0o600)
|
|
222
|
+
|
|
223
|
+
# 3. Save config.yaml in the working dir. Mounted into the allocator
|
|
224
|
+
# container at /config/config.yaml (which matches the container's
|
|
225
|
+
# CONFIG_DIR default).
|
|
226
|
+
save_config(cfg, target / "config.yaml")
|
|
227
|
+
|
|
228
|
+
# 4. Stage the custom startup script. Mirrors deploy.py:99-117 for
|
|
229
|
+
# the AWS path: ~/.lablink/custom-startup.sh wins (CLI override),
|
|
230
|
+
# else cfg.startup_script.path on the operator's filesystem. The
|
|
231
|
+
# file is always materialized (empty when disabled or absent) so
|
|
232
|
+
# the docker-compose bind mount resolves on every deploy; the
|
|
233
|
+
# allocator's registration handler only forwards it to clients
|
|
234
|
+
# when cfg.startup_script.enabled is true AND the file is non-
|
|
235
|
+
# empty.
|
|
236
|
+
# 4b. Stage the canonical-URL file. Always materialized (empty when the
|
|
237
|
+
# deployment isn't Funnel-exposed) so the compose bind mount resolves
|
|
238
|
+
# on every deploy — same reason custom-startup.sh below always exists.
|
|
239
|
+
# _enable_funnel fills it in after `compose up`, since Funnel can only
|
|
240
|
+
# be turned on once the sidecar is running. An existing value is
|
|
241
|
+
# preserved across a redeploy that stays Funnel-exposed, so the window
|
|
242
|
+
# between container start and _enable_funnel doesn't fall back to
|
|
243
|
+
# request.host_url; a deployment that turns exposure off is cleared,
|
|
244
|
+
# which is what stops a stale public URL being handed to clients.
|
|
245
|
+
canonical_target = target / CANONICAL_URL_FILENAME
|
|
246
|
+
if cfg.manual.participant_exposure == "tailscale_funnel":
|
|
247
|
+
previous_url = (
|
|
248
|
+
canonical_target.read_text() if canonical_target.exists() else ""
|
|
249
|
+
)
|
|
250
|
+
canonical_target.write_text(previous_url)
|
|
251
|
+
elif cfg.manual.participant_exposure == "cloudflare_tunnel":
|
|
252
|
+
# Known up front — the admin typed it. No after-the-fact write, and
|
|
253
|
+
# no window where the allocator reports the wrong URL.
|
|
254
|
+
canonical_target.write_text(f"https://{cfg.manual.public_hostname}")
|
|
255
|
+
else:
|
|
256
|
+
canonical_target.write_text("")
|
|
257
|
+
|
|
258
|
+
startup_target = target / "custom-startup.sh"
|
|
259
|
+
if cfg.startup_script.enabled and cfg.startup_script.path:
|
|
260
|
+
user_script = Path.home() / ".lablink" / "custom-startup.sh"
|
|
261
|
+
if user_script.exists():
|
|
262
|
+
src_startup = user_script
|
|
263
|
+
else:
|
|
264
|
+
src_startup = Path(cfg.startup_script.path)
|
|
265
|
+
if src_startup.exists():
|
|
266
|
+
shutil.copy2(src_startup, startup_target)
|
|
267
|
+
else:
|
|
268
|
+
console.print(
|
|
269
|
+
f"[yellow]startup_script.enabled=true but {src_startup} "
|
|
270
|
+
"not found — continuing without it.[/yellow]"
|
|
271
|
+
)
|
|
272
|
+
startup_target.touch()
|
|
273
|
+
else:
|
|
274
|
+
startup_target.touch()
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _allocator_image(cfg: Config) -> str:
|
|
278
|
+
"""Construct the full allocator image string from base + image_tag.
|
|
279
|
+
|
|
280
|
+
The canonical config exposes only image_tag (e.g.,
|
|
281
|
+
"linux-amd64-latest"); the registry/repo is fixed for now.
|
|
282
|
+
"""
|
|
283
|
+
tag = getattr(cfg.allocator, "image_tag", None) or "linux-amd64-latest"
|
|
284
|
+
return f"{ALLOCATOR_IMAGE_BASE}:{tag}"
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def run_deploy_compose(
|
|
288
|
+
cfg: Config,
|
|
289
|
+
*,
|
|
290
|
+
yes: bool = False,
|
|
291
|
+
workdir_root: Path | None = None,
|
|
292
|
+
tailscale_authkey: str | None = None,
|
|
293
|
+
cloudflare_tunnel_token: str | None = None,
|
|
294
|
+
docker: Docker | None = None,
|
|
295
|
+
) -> None:
|
|
296
|
+
"""Bring up the allocator stack via docker-compose.
|
|
297
|
+
|
|
298
|
+
Renders the compose working directory (`compose_workdir(cfg)`),
|
|
299
|
+
runs `docker compose up -d`, polls the allocator's `/api/health`
|
|
300
|
+
endpoint until it reports healthy (or times out), and prints a
|
|
301
|
+
summary including the register-token used by BYO clients.
|
|
302
|
+
|
|
303
|
+
`yes=True` skips the interactive confirmation prompt.
|
|
304
|
+
`workdir_root` overrides `DEFAULT_COMPOSE_DIR` (used by tests).
|
|
305
|
+
`tailscale_authkey` is required when a tailnet join is needed for
|
|
306
|
+
either `cfg.manual.connectivity == "mesh_overlay"` or
|
|
307
|
+
`cfg.manual.participant_exposure == "tailscale_funnel"`, unless a
|
|
308
|
+
value is already on record in this deployment's existing `.env`
|
|
309
|
+
(carried forward on ordinary redeploys by `render_compose_dir`) or
|
|
310
|
+
the sidecar already has a valid, authenticated identity sitting in
|
|
311
|
+
its preserved `tailscale_state` volume (e.g. after a default
|
|
312
|
+
`lablink destroy`, which wipes the working directory — including
|
|
313
|
+
`.env` — but keeps that volume specifically so this doesn't force a
|
|
314
|
+
needless re-auth).
|
|
315
|
+
`cloudflare_tunnel_token` is required when
|
|
316
|
+
`cfg.manual.participant_exposure == "cloudflare_tunnel"`, unless a
|
|
317
|
+
value is already on record in this deployment's existing `.env`. There
|
|
318
|
+
is no state-volume equivalent here: the tunnel's identity lives in
|
|
319
|
+
Cloudflare's account, and the token is the only local copy.
|
|
320
|
+
"""
|
|
321
|
+
docker = docker or default_docker()
|
|
322
|
+
target = compose_workdir(cfg, workdir_root)
|
|
323
|
+
|
|
324
|
+
needs_sidecar = _needs_tailscale_sidecar(cfg)
|
|
325
|
+
if needs_sidecar:
|
|
326
|
+
# Checking ".env exists" alone (i.e. "is this a redeploy") isn't
|
|
327
|
+
# enough: a redeploy that *switches* to needing the sidecar has
|
|
328
|
+
# an existing .env, but that .env has no TS_AUTHKEY line to carry
|
|
329
|
+
# forward. Read the actual prior value (if any) so that case
|
|
330
|
+
# still requires --tailscale-authkey instead of silently
|
|
331
|
+
# rendering an empty key.
|
|
332
|
+
previous_authkey = _read_env_value(target / ".env", "TS_AUTHKEY")
|
|
333
|
+
if (
|
|
334
|
+
not tailscale_authkey
|
|
335
|
+
and not previous_authkey
|
|
336
|
+
and not _tailscale_state_volume_exists(target, docker=docker)
|
|
337
|
+
):
|
|
338
|
+
console.print(
|
|
339
|
+
"[red]A Tailscale sidecar is needed (manual.connectivity "
|
|
340
|
+
"is 'mesh_overlay' and/or manual.participant_exposure is "
|
|
341
|
+
"'tailscale_funnel') but no --tailscale-authkey was given, "
|
|
342
|
+
"and no previous value is on record for this "
|
|
343
|
+
"deployment.[/red]\n"
|
|
344
|
+
"Generate an authkey from your Tailscale admin console "
|
|
345
|
+
"and re-run with --tailscale-authkey <key>."
|
|
346
|
+
)
|
|
347
|
+
raise SystemExit(1)
|
|
348
|
+
|
|
349
|
+
# Preflight: cloudflare_tunnel needs a hostname and a token. The
|
|
350
|
+
# hostname is also checked by get_config_errors(), but `lablink deploy`
|
|
351
|
+
# never calls that validator for the manual provider — this is the
|
|
352
|
+
# actual enforcement point for a hand-edited config.yaml. Modeled on
|
|
353
|
+
# the TS_AUTHKEY check above, minus the state-volume clause: there is
|
|
354
|
+
# no volume here, so "on record in .env" is the whole condition.
|
|
355
|
+
if cfg.manual.participant_exposure == "cloudflare_tunnel":
|
|
356
|
+
if not cfg.manual.public_hostname:
|
|
357
|
+
console.print(
|
|
358
|
+
"[red]manual.participant_exposure is 'cloudflare_tunnel' but "
|
|
359
|
+
"manual.public_hostname is empty.[/red]\n"
|
|
360
|
+
"Set it to the hostname you configured as the tunnel's "
|
|
361
|
+
"public hostname in Cloudflare (e.g. lab.smithlab.org)."
|
|
362
|
+
)
|
|
363
|
+
raise SystemExit(1)
|
|
364
|
+
from lablink_allocator_service.validate_config import (
|
|
365
|
+
PUBLIC_HOSTNAME_HINT,
|
|
366
|
+
is_valid_public_hostname,
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
# The value is interpolated into "https://{host}" for the canonical-URL
|
|
370
|
+
# file clients are handed, and canonical_base_url accepts anything that
|
|
371
|
+
# merely startswith("https://") — so a pasted scheme yields
|
|
372
|
+
# "https://https://host" that fails silently rather than loudly.
|
|
373
|
+
if not is_valid_public_hostname(cfg.manual.public_hostname):
|
|
374
|
+
console.print(
|
|
375
|
+
"[red]manual.public_hostname is not a bare hostname.[/red]\n"
|
|
376
|
+
f"It {PUBLIC_HOSTNAME_HINT}.",
|
|
377
|
+
highlight=False,
|
|
378
|
+
)
|
|
379
|
+
console.print(f" got: {cfg.manual.public_hostname!r}", highlight=False)
|
|
380
|
+
raise SystemExit(1)
|
|
381
|
+
previous_cf_token = _read_env_value(target / ".env", "CLOUDFLARE_TUNNEL_TOKEN")
|
|
382
|
+
if not cloudflare_tunnel_token and not previous_cf_token:
|
|
383
|
+
console.print(
|
|
384
|
+
"[red]manual.participant_exposure is 'cloudflare_tunnel' but "
|
|
385
|
+
"no --cloudflare-tunnel-token was given, and no previous "
|
|
386
|
+
"value is on record for this deployment.[/red]\n"
|
|
387
|
+
"Create a tunnel in Cloudflare's Zero Trust dashboard "
|
|
388
|
+
"(Networks > Tunnels), copy the token from its Docker "
|
|
389
|
+
"install command, and re-run with "
|
|
390
|
+
"--cloudflare-tunnel-token <token>."
|
|
391
|
+
)
|
|
392
|
+
raise SystemExit(1)
|
|
393
|
+
|
|
394
|
+
# Preflight: SSL provider must be one the compose template supports.
|
|
395
|
+
# The allocator image has no TLS terminator, so only ssl=none works
|
|
396
|
+
# out of the box. Operators who need TLS run their own reverse proxy
|
|
397
|
+
# in front of the compose stack.
|
|
398
|
+
if cfg.ssl.provider not in SUPPORTED_SSL_FOR_MANUAL:
|
|
399
|
+
console.print(
|
|
400
|
+
f"[red]Manual provider deploy supports only "
|
|
401
|
+
f"ssl.provider='none' (got '{cfg.ssl.provider}').[/red]\n"
|
|
402
|
+
"The allocator image has no TLS terminator; for public TLS, "
|
|
403
|
+
"front the compose stack with your own reverse proxy "
|
|
404
|
+
"(Caddy, nginx, Cloudflare Tunnel)."
|
|
405
|
+
)
|
|
406
|
+
raise SystemExit(1)
|
|
407
|
+
|
|
408
|
+
# Preflight: lan_direct + any public exposure is not a supported
|
|
409
|
+
# combination. lan_direct sends the participant's browser straight to
|
|
410
|
+
# a client's LAN IP (ws://<client-ip>:6080 — see
|
|
411
|
+
# LANDirectClientConnectivity), bypassing the allocator entirely —
|
|
412
|
+
# unreachable off-LAN and blocked as mixed content once the allocator
|
|
413
|
+
# itself is publicly exposed. mesh_overlay proxies sessions through the
|
|
414
|
+
# allocator's own nginx instead, which any exposure mode publishes.
|
|
415
|
+
# get_config_errors() also rejects this (catches it in the wizard/
|
|
416
|
+
# `show-config`/`doctor`), but `lablink deploy` never calls that
|
|
417
|
+
# validator for the manual provider — this is the actual enforcement
|
|
418
|
+
# point for a hand-edited config.yaml deployed directly.
|
|
419
|
+
if (
|
|
420
|
+
cfg.manual.participant_exposure != "none"
|
|
421
|
+
and cfg.manual.connectivity == "lan_direct"
|
|
422
|
+
):
|
|
423
|
+
console.print(
|
|
424
|
+
f"[red]manual.participant_exposure is "
|
|
425
|
+
f"'{cfg.manual.participant_exposure}' but manual.connectivity is "
|
|
426
|
+
f"'lan_direct'.[/red]\n"
|
|
427
|
+
"Participant sessions would connect directly to a client's LAN "
|
|
428
|
+
"IP, which is unreachable off-LAN and blocked as mixed content "
|
|
429
|
+
"from the HTTPS page. Use manual.connectivity: mesh_overlay "
|
|
430
|
+
"instead, which proxies sessions through the allocator."
|
|
431
|
+
)
|
|
432
|
+
raise SystemExit(1)
|
|
433
|
+
|
|
434
|
+
# Preflight: docker on PATH.
|
|
435
|
+
try:
|
|
436
|
+
docker.require()
|
|
437
|
+
except DockerUnavailable:
|
|
438
|
+
console.print(
|
|
439
|
+
"[red]docker not found on PATH.[/red] "
|
|
440
|
+
"Install Docker Engine + the Compose plugin "
|
|
441
|
+
"(https://docs.docker.com/engine/install/) and re-run."
|
|
442
|
+
)
|
|
443
|
+
raise SystemExit(1)
|
|
444
|
+
|
|
445
|
+
# Resolve admin credentials (mirrors AWS deploy.py). The wizard does
|
|
446
|
+
# NOT collect admin user/password — they're resolved here. Write the
|
|
447
|
+
# resolved values back to cfg so render_compose_dir picks them up
|
|
448
|
+
# via cfg.app.admin_user / cfg.app.admin_password.
|
|
449
|
+
admin_user, admin_pw = resolve_admin_credentials(cfg)
|
|
450
|
+
cfg.app.admin_user = admin_user
|
|
451
|
+
cfg.app.admin_password = admin_pw
|
|
452
|
+
|
|
453
|
+
# Preflight: a publicly exposed allocator is scanned by bots within
|
|
454
|
+
# minutes of publication (empirically confirmed 2026-07-22) — refuse
|
|
455
|
+
# to ship a weak/example admin password once that's the case. Placed
|
|
456
|
+
# after resolve_admin_credentials so a value resolved interactively
|
|
457
|
+
# is what actually gets checked, not whatever (possibly empty) value
|
|
458
|
+
# cfg.app.admin_password held before resolution.
|
|
459
|
+
if cfg.manual.participant_exposure != "none":
|
|
460
|
+
from lablink_allocator_service.validate_config import is_weak_admin_password
|
|
461
|
+
|
|
462
|
+
if is_weak_admin_password(admin_pw):
|
|
463
|
+
console.print(
|
|
464
|
+
f"[red]manual.participant_exposure is "
|
|
465
|
+
f"'{cfg.manual.participant_exposure}' but the resolved admin "
|
|
466
|
+
"password is empty, a known example value, or shorter than "
|
|
467
|
+
"12 characters.[/red]\n"
|
|
468
|
+
"A publicly exposed allocator is reachable from the internet "
|
|
469
|
+
"and gets scanned within minutes — set a strong "
|
|
470
|
+
"admin_password (12+ characters, not a common default) "
|
|
471
|
+
"before deploying."
|
|
472
|
+
)
|
|
473
|
+
raise SystemExit(1)
|
|
474
|
+
|
|
475
|
+
if not yes:
|
|
476
|
+
action = "create" if not target.exists() else "update"
|
|
477
|
+
console.print(
|
|
478
|
+
f"About to {action} compose stack in {target}\n"
|
|
479
|
+
f" provider: manual\n"
|
|
480
|
+
f" ssl: {cfg.ssl.provider}\n"
|
|
481
|
+
f" admin user: {admin_user}\n"
|
|
482
|
+
)
|
|
483
|
+
if not typer.confirm("Proceed?", default=True):
|
|
484
|
+
console.print("Aborted.")
|
|
485
|
+
raise SystemExit(1)
|
|
486
|
+
|
|
487
|
+
# Initialize deployment metrics — written incrementally so a failed
|
|
488
|
+
# deploy still leaves a record on disk, same as the AWS path. Started
|
|
489
|
+
# here, after the confirmation gate, so an aborted "Proceed?" leaves no
|
|
490
|
+
# in_progress file behind at all. region/template_version stay None:
|
|
491
|
+
# there is no region and no OpenTofu template in a compose deploy.
|
|
492
|
+
deploy_start_dt = datetime.now(timezone.utc)
|
|
493
|
+
metrics = DeploymentMetrics(
|
|
494
|
+
deployment_name=cfg.deployment_name,
|
|
495
|
+
provider="manual",
|
|
496
|
+
ssl_enabled=cfg.ssl.provider != "none",
|
|
497
|
+
allocator_deploy_start_time=deploy_start_dt.isoformat(),
|
|
498
|
+
)
|
|
499
|
+
metrics_path = cache_path_for(cfg.deployment_name, deploy_start_dt)
|
|
500
|
+
write_metrics(metrics_path, metrics)
|
|
501
|
+
|
|
502
|
+
# Everything from here to the success write is inside the try: the record
|
|
503
|
+
# already exists on disk, so any escape that skips the write below strands
|
|
504
|
+
# it at in_progress — indistinguishable from a Ctrl-C, and with null
|
|
505
|
+
# timings. render_compose_dir writes files and calls save_config, and
|
|
506
|
+
# _write_canonical_url writes one too, so OSError is live in both stretches
|
|
507
|
+
# either side of the timed phases, not just in the phases themselves.
|
|
508
|
+
try:
|
|
509
|
+
render_compose_dir(
|
|
510
|
+
cfg,
|
|
511
|
+
target,
|
|
512
|
+
tailscale_authkey=tailscale_authkey,
|
|
513
|
+
cloudflare_tunnel_token=cloudflare_tunnel_token,
|
|
514
|
+
)
|
|
515
|
+
console.print(f"[green]Rendered {target}[/green]")
|
|
516
|
+
|
|
517
|
+
# Explicitly disable Funnel *before* _compose_up, whenever the new
|
|
518
|
+
# config no longer wants it — this must run before --remove-orphans
|
|
519
|
+
# potentially deletes the sidecar (a removed container can't be
|
|
520
|
+
# `docker exec`'d into), and it's needed even when the sidecar
|
|
521
|
+
# sticks around unchanged (e.g. connectivity=mesh_overlay alone),
|
|
522
|
+
# since Funnel persists in the sidecar's own state regardless of
|
|
523
|
+
# whether _enable_funnel keeps getting called. See _disable_funnel.
|
|
524
|
+
if cfg.manual.participant_exposure != "tailscale_funnel":
|
|
525
|
+
_disable_funnel(docker=docker)
|
|
526
|
+
|
|
527
|
+
with phase_timer(
|
|
528
|
+
metrics, "allocator_compose_up_duration_seconds", metrics_path
|
|
529
|
+
):
|
|
530
|
+
_compose_up(target, docker=docker)
|
|
531
|
+
with phase_timer(
|
|
532
|
+
metrics, "allocator_health_check_duration_seconds", metrics_path
|
|
533
|
+
):
|
|
534
|
+
_health_poll(docker=docker)
|
|
535
|
+
|
|
536
|
+
# Disable again, now that the sidecar (if the compose file still
|
|
537
|
+
# declares one) is guaranteed running. The call above can silently
|
|
538
|
+
# no-op if the sidecar was stopped-but-not-removed at that point —
|
|
539
|
+
# `docker exec` fails on a stopped container the same way it does on
|
|
540
|
+
# a missing one, and _disable_funnel() can't tell those apart. If
|
|
541
|
+
# connectivity stays mesh_overlay, _compose_up just restarted that
|
|
542
|
+
# same stopped sidecar, reattached to tailscale_state with Funnel's
|
|
543
|
+
# last-known "on" config still intact — this second call is what
|
|
544
|
+
# actually clears it. Harmless no-op if the sidecar was removed as
|
|
545
|
+
# an orphan instead (nothing to disable).
|
|
546
|
+
if cfg.manual.participant_exposure != "tailscale_funnel":
|
|
547
|
+
_disable_funnel(docker=docker)
|
|
548
|
+
|
|
549
|
+
funnel_ok = True
|
|
550
|
+
funnel_url = None
|
|
551
|
+
if cfg.manual.participant_exposure == "tailscale_funnel":
|
|
552
|
+
funnel_ok, funnel_url = _enable_funnel(docker=docker)
|
|
553
|
+
if funnel_url:
|
|
554
|
+
_write_canonical_url(target, funnel_url)
|
|
555
|
+
|
|
556
|
+
if cfg.manual.participant_exposure == "cloudflare_tunnel":
|
|
557
|
+
if not _verify_public_hostname(cfg.manual.public_hostname):
|
|
558
|
+
console.print(
|
|
559
|
+
f"[yellow]https://{cfg.manual.public_hostname} did not "
|
|
560
|
+
"answer.[/yellow]\n"
|
|
561
|
+
"The stack is up locally. Common causes: the DNS record is "
|
|
562
|
+
"still propagating (retry in a few minutes), or the "
|
|
563
|
+
"tunnel's public hostname in Cloudflare does not point at "
|
|
564
|
+
"http://localhost:5000."
|
|
565
|
+
)
|
|
566
|
+
_print_last_log_lines(docker=docker)
|
|
567
|
+
except (Exception, SystemExit) as e:
|
|
568
|
+
# SystemExit IS caught here, unlike the AWS path where it means "user
|
|
569
|
+
# cancelled" and in_progress is the honest state. Nothing in this
|
|
570
|
+
# stretch is a cancellation — _compose_up and _health_poll both raise
|
|
571
|
+
# SystemExit for a genuine failure (non-zero compose exit, health
|
|
572
|
+
# timeout), so leaving those as in_progress would under-report real
|
|
573
|
+
# failures. KeyboardInterrupt is a BaseException and still escapes.
|
|
574
|
+
metrics.status = "failed"
|
|
575
|
+
metrics.error = str(e)
|
|
576
|
+
write_metrics(metrics_path, metrics)
|
|
577
|
+
raise
|
|
578
|
+
|
|
579
|
+
funnel_active = cfg.manual.participant_exposure == "tailscale_funnel" and funnel_ok
|
|
580
|
+
|
|
581
|
+
# Total = sum of the timed phases, matching the AWS path's definition
|
|
582
|
+
# (machine work only, excluding prompt time). Exposure setup — Funnel
|
|
583
|
+
# enable, public-hostname verification — is deliberately untimed, so a
|
|
584
|
+
# slow DNS propagation doesn't masquerade as slow deploy machinery.
|
|
585
|
+
metrics.allocator_deploy_end_time = datetime.now(timezone.utc).isoformat()
|
|
586
|
+
metrics.allocator_total_deployment_duration_seconds = round(
|
|
587
|
+
sum(
|
|
588
|
+
v
|
|
589
|
+
for v in (
|
|
590
|
+
metrics.allocator_compose_up_duration_seconds,
|
|
591
|
+
metrics.allocator_health_check_duration_seconds,
|
|
592
|
+
)
|
|
593
|
+
if v is not None
|
|
594
|
+
),
|
|
595
|
+
3,
|
|
596
|
+
)
|
|
597
|
+
metrics.status = "success" if funnel_ok else "failed"
|
|
598
|
+
if not funnel_ok:
|
|
599
|
+
metrics.error = "tailscale funnel could not be enabled"
|
|
600
|
+
write_metrics(metrics_path, metrics)
|
|
601
|
+
|
|
602
|
+
_print_summary(
|
|
603
|
+
cfg, funnel_active=funnel_active, funnel_url=funnel_url, docker=docker
|
|
604
|
+
)
|
|
605
|
+
|
|
606
|
+
if not funnel_ok:
|
|
607
|
+
raise SystemExit(1)
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def _verify_public_hostname(hostname: str) -> bool:
|
|
611
|
+
"""True if the allocator answers on its public hostname.
|
|
612
|
+
|
|
613
|
+
One request over the real public URL proves DNS resolution, the
|
|
614
|
+
Cloudflare edge, the tunnel, nginx and Flask together — checking that
|
|
615
|
+
cloudflared is alive says nothing about whether the edge found it.
|
|
616
|
+
|
|
617
|
+
Polled, because the local `_health_poll` above clears as soon as Flask
|
|
618
|
+
answers — which is *before* the public path exists. cloudflared is still
|
|
619
|
+
registering its edge connections at that point and nginx is still binding
|
|
620
|
+
:5000, so a single attempt fails on a perfectly good deploy (observed
|
|
621
|
+
live 2026-08-05: warned at 20:25:30, the same container served the public
|
|
622
|
+
hostname seconds later and stayed up).
|
|
623
|
+
|
|
624
|
+
Still advisory, and bounded short: the remaining failure modes — a
|
|
625
|
+
still-propagating DNS record, or a public hostname in the Cloudflare
|
|
626
|
+
dashboard pointing somewhere other than the origin — are not things
|
|
627
|
+
waiting fixes, and the caller only warns.
|
|
628
|
+
"""
|
|
629
|
+
url = f"https://{hostname}"
|
|
630
|
+
console.print(
|
|
631
|
+
f"[bold]Verifying public hostname {url}/api/health "
|
|
632
|
+
f"(up to {PUBLIC_HOSTNAME_MAX_ATTEMPTS} tries) …[/bold]"
|
|
633
|
+
)
|
|
634
|
+
for attempt in range(1, PUBLIC_HOSTNAME_MAX_ATTEMPTS + 1):
|
|
635
|
+
try:
|
|
636
|
+
healthy = bool(check_health_endpoint(url).get("healthy"))
|
|
637
|
+
except OSError:
|
|
638
|
+
# Unresolvable name / refused connection while the record
|
|
639
|
+
# propagates, or while the tunnel route is still coming up.
|
|
640
|
+
healthy = False
|
|
641
|
+
if healthy:
|
|
642
|
+
console.print(f"[green]Public hostname is live: {url}[/green]")
|
|
643
|
+
return True
|
|
644
|
+
if attempt < PUBLIC_HOSTNAME_MAX_ATTEMPTS:
|
|
645
|
+
time.sleep(PUBLIC_HOSTNAME_RETRY_DELAY_SECONDS)
|
|
646
|
+
return False
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
def _compose_up(target: Path, *, docker: Docker) -> None:
|
|
650
|
+
console.print("[bold]docker compose up -d …[/bold]")
|
|
651
|
+
# --remove-orphans: if needs_sidecar just became False (connectivity
|
|
652
|
+
# switched off mesh_overlay AND participant_exposure switched off
|
|
653
|
+
# tailscale_funnel), the freshly-rendered compose file no longer
|
|
654
|
+
# declares the tailscale service — without this flag, `docker
|
|
655
|
+
# compose up` leaves that now-undeclared container running
|
|
656
|
+
# untouched, forever. _disable_funnel() (called before this, in
|
|
657
|
+
# run_deploy_compose) already clears its Funnel state first, so
|
|
658
|
+
# this just ensures the container itself doesn't linger too.
|
|
659
|
+
result = docker.compose(target, "up", "-d", "--remove-orphans", capture=False)
|
|
660
|
+
if not result.ok:
|
|
661
|
+
console.print("[red]docker compose up failed.[/red]")
|
|
662
|
+
raise SystemExit(result.returncode or 1)
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
def _write_canonical_url(target: Path, url: str) -> None:
|
|
666
|
+
"""Publish the allocator's real public URL to the bind-mounted file the
|
|
667
|
+
allocator reads (see config_helpers.canonical_base_url).
|
|
668
|
+
|
|
669
|
+
Behind Funnel the allocator cannot work its own public URL out from the
|
|
670
|
+
request: Funnel injects no X-Forwarded-Proto, and manual deployments run
|
|
671
|
+
ssl.provider=none so the header-trust gate is shut anyway. It therefore
|
|
672
|
+
reports http://, which clients can only follow via a 302 that downgrades
|
|
673
|
+
their POSTs to GET. This file is the out-of-band channel that fixes that,
|
|
674
|
+
carrying the address `tailscale funnel status` actually reported — which
|
|
675
|
+
also picks up the numeric hostname suffixes (-2, -3, ...) that a name
|
|
676
|
+
collision with an offline node from a prior deploy produces.
|
|
677
|
+
|
|
678
|
+
Written IN PLACE, never via a temp file + rename: docker bind-mounts a
|
|
679
|
+
single file by inode, so a rename would leave the running container
|
|
680
|
+
reading the old file forever.
|
|
681
|
+
"""
|
|
682
|
+
path = target / CANONICAL_URL_FILENAME
|
|
683
|
+
with path.open("w") as f:
|
|
684
|
+
f.write(f"{url.rstrip('/')}\n")
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
FUNNEL_STATUS_URL_RE = re.compile(r"(https://\S+)\s*\(Funnel on\)")
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
def _funnel_status_url(*, docker: Docker) -> str | None:
|
|
691
|
+
"""Query the sidecar for the public URL Tailscale Funnel is actually
|
|
692
|
+
serving right now, via `tailscale funnel status`.
|
|
693
|
+
|
|
694
|
+
This is the authoritative source for the URL — Tailscale assigns the
|
|
695
|
+
node's hostname, and it does not necessarily match
|
|
696
|
+
`lablink-allocator-<deployment_name>`: a name collision with an
|
|
697
|
+
existing (possibly offline) tailnet node from a prior deploy gets a
|
|
698
|
+
numeric suffix appended instead (verified live: `-2`, `-3`, ... after
|
|
699
|
+
repeated deploy/destroy cycles). Returns None if Funnel isn't active
|
|
700
|
+
or the output didn't match the expected format.
|
|
701
|
+
"""
|
|
702
|
+
result = docker.exec_in(
|
|
703
|
+
TAILSCALE_SIDECAR_CONTAINER_NAME,
|
|
704
|
+
["tailscale", "funnel", "status"],
|
|
705
|
+
)
|
|
706
|
+
match = FUNNEL_STATUS_URL_RE.search(result.stdout)
|
|
707
|
+
return match.group(1) if match else None
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def _enable_funnel(*, docker: Docker) -> tuple[bool, str | None]:
|
|
711
|
+
"""Idempotently enable Tailscale Funnel on the allocator's own nginx
|
|
712
|
+
port, via the sidecar container.
|
|
713
|
+
|
|
714
|
+
`tailscale funnel`'s config lives in tailscaled's own local state
|
|
715
|
+
(already persisted by the compose file's `tailscale_state` named
|
|
716
|
+
volume), so this is safe to re-run on every deploy — a no-op if
|
|
717
|
+
already enabled. If the tailnet hasn't granted the Funnel ACL yet,
|
|
718
|
+
the command's own output names the exact grant URL; this surfaces
|
|
719
|
+
that URL and returns (False, None) rather than silently leaving the
|
|
720
|
+
allocator unreachable to participants.
|
|
721
|
+
|
|
722
|
+
Retries a few times with a short delay: the sidecar may still be
|
|
723
|
+
completing its own `tailscale up` join when this runs (right after
|
|
724
|
+
`_compose_up`/`_health_poll`, which only confirm the *allocator*
|
|
725
|
+
container is healthy, not the sidecar's tailnet membership) — the
|
|
726
|
+
same class of startup race already fixed on the client side (commit
|
|
727
|
+
7a8ab9f6). Only retried for transient not-ready-yet failures; an
|
|
728
|
+
ACL-not-granted response is unambiguous and returned immediately
|
|
729
|
+
without retrying, since retrying can't fix a missing grant.
|
|
730
|
+
|
|
731
|
+
Returns (True, url) if Funnel is enabled (or already was) — url is
|
|
732
|
+
the real address from `_funnel_status_url()`, or None if that lookup
|
|
733
|
+
didn't find one despite the enable itself succeeding. Returns
|
|
734
|
+
(False, None) otherwise (ACL not granted, or failure persisting
|
|
735
|
+
across all retries) — callers should still let the rest of the
|
|
736
|
+
deploy complete either way (the stack is functional for LAN/
|
|
737
|
+
mesh-overlay access regardless), but should ultimately exit non-zero
|
|
738
|
+
when this returns False.
|
|
739
|
+
"""
|
|
740
|
+
for attempt in range(1, FUNNEL_ENABLE_MAX_ATTEMPTS + 1):
|
|
741
|
+
result = docker.exec_in(
|
|
742
|
+
TAILSCALE_SIDECAR_CONTAINER_NAME,
|
|
743
|
+
["tailscale", "funnel", "--bg", str(ALLOCATOR_INTERNAL_PORT)],
|
|
744
|
+
)
|
|
745
|
+
output = result.stdout + result.stderr
|
|
746
|
+
if FUNNEL_ACL_NOT_GRANTED_MARKER in output:
|
|
747
|
+
console.print(
|
|
748
|
+
"[yellow]Tailscale Funnel isn't authorized on this tailnet "
|
|
749
|
+
"yet.[/yellow] The compose stack is up and reachable on your "
|
|
750
|
+
"LAN, but participant exposure needs a one-time grant:\n"
|
|
751
|
+
)
|
|
752
|
+
console.print(output.strip())
|
|
753
|
+
return False, None
|
|
754
|
+
if result.returncode == 0:
|
|
755
|
+
console.print(
|
|
756
|
+
"[green]Tailscale Funnel enabled for participant access.[/green]"
|
|
757
|
+
)
|
|
758
|
+
console.print(output.strip())
|
|
759
|
+
return True, _funnel_status_url(docker=docker)
|
|
760
|
+
if attempt < FUNNEL_ENABLE_MAX_ATTEMPTS:
|
|
761
|
+
time.sleep(FUNNEL_ENABLE_RETRY_DELAY_SECONDS)
|
|
762
|
+
continue
|
|
763
|
+
console.print(
|
|
764
|
+
f"[red]Failed to enable Tailscale Funnel after "
|
|
765
|
+
f"{FUNNEL_ENABLE_MAX_ATTEMPTS} attempts (exit "
|
|
766
|
+
f"{result.returncode}):[/red]\n{output.strip()}"
|
|
767
|
+
)
|
|
768
|
+
return False, None
|
|
769
|
+
|
|
770
|
+
|
|
771
|
+
def _disable_funnel(*, docker: Docker) -> None:
|
|
772
|
+
"""Explicitly clear Tailscale Funnel's serve config on the sidecar,
|
|
773
|
+
best-effort.
|
|
774
|
+
|
|
775
|
+
`tailscale funnel --bg` persists in tailscaled's own local state (the
|
|
776
|
+
`tailscale_state` named volume) across container restarts — and even
|
|
777
|
+
across container *recreation*, since a freshly-created sidecar
|
|
778
|
+
reattaches to that same volume and the same node identity resumes
|
|
779
|
+
serving from its last-known config. Simply no longer calling
|
|
780
|
+
`_enable_funnel()` is NOT enough to actually turn Funnel off; it has
|
|
781
|
+
to be explicitly disabled, or a previously-Funnel-exposed allocator
|
|
782
|
+
stays publicly reachable even after an operator sets
|
|
783
|
+
participant_exposure back to "none".
|
|
784
|
+
|
|
785
|
+
Called whenever the new config's participant_exposure is no longer
|
|
786
|
+
"tailscale_funnel", *before* `_compose_up` — including the case
|
|
787
|
+
where the sidecar is about to be removed entirely as a compose
|
|
788
|
+
orphan (needs_sidecar became False), since a removed container can
|
|
789
|
+
no longer be `docker exec`'d into and its persisted volume would
|
|
790
|
+
otherwise carry the stale "enabled" state forward to any future
|
|
791
|
+
sidecar that reattaches to it.
|
|
792
|
+
|
|
793
|
+
Best-effort and silent on failure: if the sidecar container doesn't
|
|
794
|
+
exist at all (e.g. a fresh deployment that never enabled Funnel),
|
|
795
|
+
there is nothing to disable and no error is surfaced.
|
|
796
|
+
"""
|
|
797
|
+
result = docker.exec_in(
|
|
798
|
+
TAILSCALE_SIDECAR_CONTAINER_NAME,
|
|
799
|
+
["tailscale", "funnel", "--https=443", "off"],
|
|
800
|
+
)
|
|
801
|
+
if result.ok:
|
|
802
|
+
console.print("[dim]Tailscale Funnel disabled.[/dim]")
|
|
803
|
+
|
|
804
|
+
|
|
805
|
+
def _health_poll(*, docker: Docker) -> None:
|
|
806
|
+
"""Poll the allocator's /api/health on localhost until healthy."""
|
|
807
|
+
# Manual provider is HTTP-only; the host port comes from the rendered
|
|
808
|
+
# .env, which defaults to DEFAULT_HTTP_PORT.
|
|
809
|
+
base_url = f"http://localhost:{DEFAULT_HTTP_PORT}"
|
|
810
|
+
|
|
811
|
+
console.print(
|
|
812
|
+
f"[bold]Polling allocator health at {base_url}/api/health "
|
|
813
|
+
f"(up to {HEALTH_POLL_TIMEOUT_SECONDS}s) …[/bold]"
|
|
814
|
+
)
|
|
815
|
+
start = time.monotonic()
|
|
816
|
+
deadline = start + HEALTH_POLL_TIMEOUT_SECONDS
|
|
817
|
+
while time.monotonic() < deadline:
|
|
818
|
+
result = check_health_endpoint(base_url)
|
|
819
|
+
if result.get("healthy"):
|
|
820
|
+
elapsed = time.monotonic() - start
|
|
821
|
+
console.print(f"[green]Allocator healthy after {elapsed:.0f}s[/green]")
|
|
822
|
+
return
|
|
823
|
+
time.sleep(3)
|
|
824
|
+
|
|
825
|
+
console.print(
|
|
826
|
+
"[yellow]Allocator did not become healthy within "
|
|
827
|
+
f"{HEALTH_POLL_TIMEOUT_SECONDS}s.[/yellow]"
|
|
828
|
+
)
|
|
829
|
+
_print_last_log_lines(docker=docker)
|
|
830
|
+
raise SystemExit(1)
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def _redact_secrets(text: str) -> str:
|
|
834
|
+
"""Blank out credential values in container output before printing it.
|
|
835
|
+
|
|
836
|
+
Keyed on the variable *name*, not the value's shape: a Cloudflare token
|
|
837
|
+
is base64-ish and a tailnet auth key is `tskey-`-prefixed, but both are
|
|
838
|
+
the vendor's to change, whereas the names are ours.
|
|
839
|
+
|
|
840
|
+
Needed because `cloudflared` logs its whole environment at INFO on
|
|
841
|
+
startup. `start.sh` unsets the token before launching it, so a current
|
|
842
|
+
image never logs it — this is the second layer, covering images built
|
|
843
|
+
before that change and any other path that echoes a secret.
|
|
844
|
+
"""
|
|
845
|
+
return re.sub(
|
|
846
|
+
r"((?:CLOUDFLARE_TUNNEL_TOKEN|TS_AUTHKEY)[=:]|--token[= ])\S+",
|
|
847
|
+
r"\1<redacted>",
|
|
848
|
+
text,
|
|
849
|
+
)
|
|
850
|
+
|
|
851
|
+
|
|
852
|
+
def _print_last_log_lines(lines: int = 30, *, docker: Docker) -> None:
|
|
853
|
+
# Merge stderr: the allocator's Python logging goes there (see
|
|
854
|
+
# _extract_register_token), so capturing the streams separately and
|
|
855
|
+
# printing only stdout hid the very tracebacks this dump exists to
|
|
856
|
+
# surface. Merging is also why the redaction above is load-bearing.
|
|
857
|
+
result = docker.logs(ALLOCATOR_CONTAINER_NAME, tail=lines, merge_stderr=True)
|
|
858
|
+
if result.stdout:
|
|
859
|
+
console.print("[dim]Last allocator log lines:[/dim]")
|
|
860
|
+
console.print(_redact_secrets(result.stdout))
|
|
861
|
+
|
|
862
|
+
|
|
863
|
+
def _print_summary(
|
|
864
|
+
cfg: Config,
|
|
865
|
+
*,
|
|
866
|
+
funnel_active: bool = False,
|
|
867
|
+
funnel_url: str | None = None,
|
|
868
|
+
docker: Docker,
|
|
869
|
+
) -> None:
|
|
870
|
+
register_token = _extract_register_token(docker=docker)
|
|
871
|
+
# Manual provider is HTTP-only; preflight rejects anything else.
|
|
872
|
+
local_url = "http://localhost"
|
|
873
|
+
lan_ip = _detect_lan_ip()
|
|
874
|
+
lan_url = f"http://{lan_ip}" if lan_ip else None
|
|
875
|
+
# BYO clients run on different boxes, so the register command needs
|
|
876
|
+
# an address those boxes can route to — localhost is only useful for
|
|
877
|
+
# self-registration on the operator's host. Prefer the LAN URL when
|
|
878
|
+
# we could detect one.
|
|
879
|
+
register_url = lan_url or local_url
|
|
880
|
+
|
|
881
|
+
# The deployment's one internet-reachable URL, or None. Funnel's has to
|
|
882
|
+
# be read back out of the sidecar and can be missing even when enabled;
|
|
883
|
+
# Cloudflare's is just config the admin typed. Everything downstream only
|
|
884
|
+
# cares whether such a URL exists, so both collapse into one value here
|
|
885
|
+
# rather than each exposure mode growing its own branch.
|
|
886
|
+
if funnel_active and funnel_url:
|
|
887
|
+
public_url = funnel_url
|
|
888
|
+
elif cfg.manual.participant_exposure == "cloudflare_tunnel":
|
|
889
|
+
public_url = f"https://{cfg.manual.public_hostname}"
|
|
890
|
+
else:
|
|
891
|
+
public_url = None
|
|
892
|
+
|
|
893
|
+
console.print("\n[bold green]Deployment complete.[/bold green]")
|
|
894
|
+
if funnel_active and not funnel_url:
|
|
895
|
+
console.print(
|
|
896
|
+
" Allocator URL (public): (enabled, but the URL could not be "
|
|
897
|
+
f"determined — run `docker exec {TAILSCALE_SIDECAR_CONTAINER_NAME} "
|
|
898
|
+
"tailscale funnel status` to see it)",
|
|
899
|
+
soft_wrap=True,
|
|
900
|
+
highlight=False,
|
|
901
|
+
)
|
|
902
|
+
elif public_url:
|
|
903
|
+
console.print(
|
|
904
|
+
f" Allocator URL (public): {public_url}",
|
|
905
|
+
soft_wrap=True,
|
|
906
|
+
highlight=False,
|
|
907
|
+
)
|
|
908
|
+
console.print(f" Allocator URL (local): {local_url}")
|
|
909
|
+
if lan_url:
|
|
910
|
+
console.print(f" Allocator URL (LAN): {lan_url}")
|
|
911
|
+
else:
|
|
912
|
+
# Be loud about *why* we couldn't pin a LAN address — operators
|
|
913
|
+
# who are routing through Tailscale/VPN/etc. need to know they
|
|
914
|
+
# have to substitute the right hostname themselves.
|
|
915
|
+
console.print(
|
|
916
|
+
" Allocator URL (LAN): (no LAN IP detected — pass the "
|
|
917
|
+
"operator host's reachable address manually)"
|
|
918
|
+
)
|
|
919
|
+
# public_url and lan_direct are mutually exclusive (preflight above), so
|
|
920
|
+
# this chain encodes the connectivity rule without re-reading it.
|
|
921
|
+
admin_url = public_url or lan_url or local_url
|
|
922
|
+
# soft_wrap: a real Funnel URL plus the column prefix overruns 80 cols.
|
|
923
|
+
console.print(
|
|
924
|
+
f" Admin URL: {admin_url}/admin",
|
|
925
|
+
soft_wrap=True,
|
|
926
|
+
highlight=False,
|
|
927
|
+
)
|
|
928
|
+
console.print(f" Admin user: {cfg.app.admin_user}")
|
|
929
|
+
if register_token:
|
|
930
|
+
console.print(f" Register token: {register_token}")
|
|
931
|
+
else:
|
|
932
|
+
# The allocator logs to stderr (Python `logging` default), so
|
|
933
|
+
# the recovery command MUST redirect stderr (`2>&1`) before the
|
|
934
|
+
# pipe — otherwise grep sees only the container's stdout and
|
|
935
|
+
# the user gets an empty result, same root cause as the bug this
|
|
936
|
+
# path is recovering from.
|
|
937
|
+
# soft_wrap=True keeps the docker-logs hint on a single line so
|
|
938
|
+
# the suggested command is not split mid-pipe in narrow terminals.
|
|
939
|
+
console.print(
|
|
940
|
+
" Register token: (could not parse from container "
|
|
941
|
+
"logs; fetch with `docker logs lablink-allocator 2>&1 | "
|
|
942
|
+
"grep REGISTER_TOKEN`)",
|
|
943
|
+
soft_wrap=True,
|
|
944
|
+
highlight=False,
|
|
945
|
+
)
|
|
946
|
+
|
|
947
|
+
# Print a copy-paste-ready command using the LAN URL when available
|
|
948
|
+
# (clients registering over the LAN can't reach localhost). The
|
|
949
|
+
# token-bearing line uses soft_wrap=True so narrow terminals don't
|
|
950
|
+
# insert a hard newline mid-command — that would break the
|
|
951
|
+
# operator's copy-paste.
|
|
952
|
+
mesh_overlay = cfg.manual.connectivity == "mesh_overlay"
|
|
953
|
+
reverse_tunnel = cfg.manual.connectivity == "reverse_tunnel"
|
|
954
|
+
# Both connectivity modes below mean the client isn't reachable on the
|
|
955
|
+
# allocator's own LAN — mesh_overlay via a Tailscale tailnet,
|
|
956
|
+
# reverse_tunnel by dialing out instead of accepting inbound at all.
|
|
957
|
+
off_lan = mesh_overlay or reverse_tunnel
|
|
958
|
+
# Substitute only when a real public URL exists — funnel_active can be
|
|
959
|
+
# True while funnel_url is None (enable succeeded but the status lookup
|
|
960
|
+
# didn't match), and a guessed fallback here would be exactly the wrong
|
|
961
|
+
# URL this function used to print. Gated on off_lan, not just
|
|
962
|
+
# mesh_overlay: a reverse_tunnel client behind a NAT'd/firewalled box is
|
|
963
|
+
# just as unreachable at the LAN address. Gated on public_url rather
|
|
964
|
+
# than on Funnel specifically, because an off-LAN client cannot reach
|
|
965
|
+
# the LAN address no matter which exposure mode publishes the
|
|
966
|
+
# allocator — printing one is how this hint was wrong before.
|
|
967
|
+
public_url_used = off_lan and bool(public_url)
|
|
968
|
+
# Hoisted above the mesh_overlay/reverse_tunnel branch so both off-LAN
|
|
969
|
+
# modes get the substitution — lan_direct clients genuinely are on the
|
|
970
|
+
# LAN, so their own hint below keeps using register_url as-is.
|
|
971
|
+
if public_url_used:
|
|
972
|
+
register_url = public_url
|
|
973
|
+
if mesh_overlay:
|
|
974
|
+
# A mesh-overlay client (e.g. a Run:AI-hosted workload) isn't on
|
|
975
|
+
# the allocator's LAN at all — the LAN URL above is unreachable
|
|
976
|
+
# from it regardless of whether we detected one. Whichever exposure
|
|
977
|
+
# mode is live, its public URL IS reachable from anywhere with
|
|
978
|
+
# internet access, so prefer that here.
|
|
979
|
+
console.print(
|
|
980
|
+
"\n[bold]Next step:[/bold] for each mesh-overlay client "
|
|
981
|
+
"(e.g. a Run:AI-hosted workload), open a terminal inside "
|
|
982
|
+
"that workload and run (hostname/machine-identity/GPU are "
|
|
983
|
+
"auto-detected):"
|
|
984
|
+
)
|
|
985
|
+
register_cmd = (
|
|
986
|
+
f" lablink client register --allocator-url {register_url} "
|
|
987
|
+
f"--register-token {register_token or '<token>'} "
|
|
988
|
+
"--overlay-hostname <name> --tailscale-authkey <key>"
|
|
989
|
+
)
|
|
990
|
+
elif reverse_tunnel:
|
|
991
|
+
console.print(
|
|
992
|
+
"\n[bold]Next step:[/bold] for each tunnel client (a box or "
|
|
993
|
+
"workload that can't accept inbound connections), open a "
|
|
994
|
+
"terminal inside it and run (hostname/machine-identity/GPU are "
|
|
995
|
+
"auto-detected; the tunnel's values are minted by the "
|
|
996
|
+
"allocator, so --tunnel takes no arguments):"
|
|
997
|
+
)
|
|
998
|
+
register_cmd = (
|
|
999
|
+
f" lablink client register --allocator-url {register_url} "
|
|
1000
|
+
f"--register-token {register_token or '<token>'} --tunnel"
|
|
1001
|
+
)
|
|
1002
|
+
else:
|
|
1003
|
+
console.print("\n[bold]Next step:[/bold] on each BYO box on the same LAN, run")
|
|
1004
|
+
register_cmd = (
|
|
1005
|
+
f" lablink client register --allocator-url {register_url} "
|
|
1006
|
+
f"--register-token {register_token or '<token>'}"
|
|
1007
|
+
)
|
|
1008
|
+
console.print(register_cmd, soft_wrap=True, highlight=False)
|
|
1009
|
+
if off_lan:
|
|
1010
|
+
console.print(
|
|
1011
|
+
" [dim]Registering ahead of time from elsewhere instead? "
|
|
1012
|
+
"Add --no-run-locally to print secrets for your own "
|
|
1013
|
+
"workload submission instead of running here, along with "
|
|
1014
|
+
"--hostname/--machine-identity.[/dim]"
|
|
1015
|
+
)
|
|
1016
|
+
if not lan_url and not public_url_used:
|
|
1017
|
+
# If we fell back to localhost, the printed command only works
|
|
1018
|
+
# for a BYO client *on the operator host*. Call that out so the
|
|
1019
|
+
# operator doesn't blindly hand it to a remote teammate. Doesn't
|
|
1020
|
+
# apply when the off-LAN hint above already substituted a public
|
|
1021
|
+
# URL instead of falling back to localhost.
|
|
1022
|
+
console.print(
|
|
1023
|
+
" [yellow]Note:[/yellow] the URL above is localhost — only "
|
|
1024
|
+
"valid for a BYO client running on this same machine. For "
|
|
1025
|
+
"clients on another box, substitute this host's LAN IP / "
|
|
1026
|
+
"hostname.",
|
|
1027
|
+
soft_wrap=True,
|
|
1028
|
+
highlight=False,
|
|
1029
|
+
)
|
|
1030
|
+
|
|
1031
|
+
|
|
1032
|
+
def _detect_lan_ip() -> str | None:
|
|
1033
|
+
"""Best-effort: the IPv4 address another host on the operator's LAN
|
|
1034
|
+
would use to reach this machine. Returns ``None`` if we can't pick
|
|
1035
|
+
one (no default route, only loopback configured, …).
|
|
1036
|
+
|
|
1037
|
+
Uses the kernel routing-table trick: open a UDP socket and call
|
|
1038
|
+
``connect()`` to a public IP. No packets are sent (UDP is
|
|
1039
|
+
connectionless), but the kernel resolves the route and binds the
|
|
1040
|
+
socket's local address — which we then read back via
|
|
1041
|
+
``getsockname()``. Works offline as long as a default route exists.
|
|
1042
|
+
"""
|
|
1043
|
+
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
1044
|
+
try:
|
|
1045
|
+
# 8.8.8.8 is a well-known recipe target — we only need the
|
|
1046
|
+
# kernel to pick *an* outbound interface, nothing is transmitted.
|
|
1047
|
+
s.connect(("8.8.8.8", 80))
|
|
1048
|
+
ip = s.getsockname()[0]
|
|
1049
|
+
except OSError:
|
|
1050
|
+
return None
|
|
1051
|
+
finally:
|
|
1052
|
+
s.close()
|
|
1053
|
+
|
|
1054
|
+
# A loopback or unspecified address means the operator's box doesn't
|
|
1055
|
+
# have a usable LAN interface; treat that as "no LAN IP".
|
|
1056
|
+
if not ip or ip.startswith("127.") or ip == "0.0.0.0":
|
|
1057
|
+
return None
|
|
1058
|
+
return ip
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
def _extract_register_token(*, docker: Docker) -> str | None:
|
|
1062
|
+
"""Parse the register_token from the allocator's startup logs.
|
|
1063
|
+
|
|
1064
|
+
The allocator logs `REGISTER_TOKEN=<token>` at startup (grep for
|
|
1065
|
+
`REGISTER_TOKEN=%s` in `lablink_allocator_service/main.py`).
|
|
1066
|
+
Also tolerate the `register_token = "..."` form just in case.
|
|
1067
|
+
|
|
1068
|
+
Python's `logging.basicConfig` writes to stderr, so `merge_stderr=True`
|
|
1069
|
+
is required here too — see `Docker.logs`.
|
|
1070
|
+
"""
|
|
1071
|
+
result = docker.logs(ALLOCATOR_CONTAINER_NAME, merge_stderr=True)
|
|
1072
|
+
if not result.ok:
|
|
1073
|
+
return None
|
|
1074
|
+
for pattern in (
|
|
1075
|
+
r'REGISTER_TOKEN\s*=\s*"?([A-Za-z0-9_\-]{20,})"?',
|
|
1076
|
+
r'register_token\s*=\s*"?([A-Za-z0-9_\-]{20,})"?',
|
|
1077
|
+
):
|
|
1078
|
+
m = re.search(pattern, result.stdout)
|
|
1079
|
+
if m:
|
|
1080
|
+
return m.group(1)
|
|
1081
|
+
return None
|
|
1082
|
+
|
|
1083
|
+
|
|
1084
|
+
def _pgdata_volume_name(target: Path, *, docker: Docker) -> str | None:
|
|
1085
|
+
"""Resolve the Docker volume currently backing the allocator's Postgres data.
|
|
1086
|
+
|
|
1087
|
+
Tries the running container's actual mount first (exact, no guessing).
|
|
1088
|
+
Falls back to Compose's own directory-basename project-naming
|
|
1089
|
+
convention when the container's already gone — e.g. an operator ran a
|
|
1090
|
+
manual `docker compose down` (removing containers, leaving volumes)
|
|
1091
|
+
before `lablink destroy` — verified via `docker volume inspect` before
|
|
1092
|
+
trusting it, so a wrong guess can't be silently mistaken for "nothing
|
|
1093
|
+
to remove". Guessing is safe here specifically because deployment_name
|
|
1094
|
+
(and therefore target.name) is already regex-constrained to Compose's
|
|
1095
|
+
own project-name character set (`^[a-z][a-z0-9-]*[a-z0-9]$` — see
|
|
1096
|
+
config/schema.py's DEPLOYMENT_NAME_RE), so there's no normalization
|
|
1097
|
+
mismatch to worry about.
|
|
1098
|
+
|
|
1099
|
+
Returns None only if no volume can be found by either method — i.e.
|
|
1100
|
+
this deployment never actually created one.
|
|
1101
|
+
"""
|
|
1102
|
+
name = docker.inspect_format(
|
|
1103
|
+
ALLOCATOR_CONTAINER_NAME,
|
|
1104
|
+
'{{range .Mounts}}{{if eq .Destination "/var/lib/postgresql"}}'
|
|
1105
|
+
"{{.Name}}{{end}}{{end}}",
|
|
1106
|
+
)
|
|
1107
|
+
if name:
|
|
1108
|
+
return name
|
|
1109
|
+
|
|
1110
|
+
candidate = f"{target.name}_allocator_pgdata"
|
|
1111
|
+
return candidate if docker.volume_exists(candidate) else None
|
|
1112
|
+
|
|
1113
|
+
|
|
1114
|
+
def run_destroy_compose(
|
|
1115
|
+
cfg: Config,
|
|
1116
|
+
*,
|
|
1117
|
+
yes: bool = False,
|
|
1118
|
+
keep_data: bool = False,
|
|
1119
|
+
workdir_root: Path | None = None,
|
|
1120
|
+
docker: Docker | None = None,
|
|
1121
|
+
) -> None:
|
|
1122
|
+
"""Tear down a manual-provider compose stack.
|
|
1123
|
+
|
|
1124
|
+
Default behavior: wipes the Postgres data volume (all registration
|
|
1125
|
+
history, sessions, etc.) plus the working directory. A subsequent
|
|
1126
|
+
`lablink deploy` with the same deployment_name then starts from a
|
|
1127
|
+
genuinely empty database, matching what "destroy" means for every
|
|
1128
|
+
other provider — previously the default silently preserved the old
|
|
1129
|
+
volume, so a "fresh" redeploy kept showing every client registered
|
|
1130
|
+
under a prior deployment.
|
|
1131
|
+
|
|
1132
|
+
The Postgres volume is removed by name (resolved via `_pgdata_volume_name`)
|
|
1133
|
+
rather than via `docker compose down --volumes`, which would also delete
|
|
1134
|
+
the mesh-overlay `tailscale_state` volume — that volume carries the
|
|
1135
|
+
Tailscale node's identity, not "data": wiping it forces a brand-new
|
|
1136
|
+
tailnet registration on the next deploy, which changes the node's
|
|
1137
|
+
hostname (and any Funnel URL already handed to participants) for no
|
|
1138
|
+
reason. `tailscale_state` is always preserved, independent of `keep_data`.
|
|
1139
|
+
|
|
1140
|
+
With `keep_data=True`: no volumes are touched at all, and the working
|
|
1141
|
+
directory is left in place — re-deploying with the same deployment_name
|
|
1142
|
+
restores the previous DB state instead of starting fresh. Opt into this
|
|
1143
|
+
only if that's specifically what you want (e.g. a deliberate maintenance
|
|
1144
|
+
restart, not a real teardown).
|
|
1145
|
+
|
|
1146
|
+
`yes=True` skips the interactive confirmation prompt.
|
|
1147
|
+
`workdir_root` overrides `DEFAULT_COMPOSE_DIR` (used by tests).
|
|
1148
|
+
"""
|
|
1149
|
+
docker = docker or default_docker()
|
|
1150
|
+
target = compose_workdir(cfg, workdir_root)
|
|
1151
|
+
|
|
1152
|
+
if not target.exists():
|
|
1153
|
+
console.print(
|
|
1154
|
+
f"[yellow]No compose stack at {target} — already destroyed.[/yellow]"
|
|
1155
|
+
)
|
|
1156
|
+
return
|
|
1157
|
+
|
|
1158
|
+
if not yes:
|
|
1159
|
+
if not keep_data:
|
|
1160
|
+
console.print(
|
|
1161
|
+
"[red bold]This will DELETE the Postgres data volume "
|
|
1162
|
+
"(all registration history, sessions, etc.). Pass "
|
|
1163
|
+
"--keep-data to preserve it instead.[/red bold]"
|
|
1164
|
+
)
|
|
1165
|
+
confirmation = typer.prompt(
|
|
1166
|
+
f"Type 'yes' to tear down compose stack at {target}",
|
|
1167
|
+
default="no",
|
|
1168
|
+
show_default=False,
|
|
1169
|
+
)
|
|
1170
|
+
if confirmation.strip().lower() != "yes":
|
|
1171
|
+
console.print("Aborted.")
|
|
1172
|
+
raise SystemExit(1)
|
|
1173
|
+
|
|
1174
|
+
pgdata_volume = None if keep_data else _pgdata_volume_name(target, docker=docker)
|
|
1175
|
+
|
|
1176
|
+
result = docker.compose(target, "down", capture=False)
|
|
1177
|
+
if not result.ok:
|
|
1178
|
+
console.print("[red]docker compose down failed.[/red]")
|
|
1179
|
+
raise SystemExit(result.returncode or 1)
|
|
1180
|
+
|
|
1181
|
+
if not keep_data:
|
|
1182
|
+
if pgdata_volume:
|
|
1183
|
+
rm_result = docker.remove_volume(pgdata_volume)
|
|
1184
|
+
if not rm_result.ok:
|
|
1185
|
+
console.print(
|
|
1186
|
+
f"[red]Failed to remove Postgres volume "
|
|
1187
|
+
f"{pgdata_volume}:[/red] {rm_result.stderr.strip()}\n"
|
|
1188
|
+
"The working directory was NOT removed — a later "
|
|
1189
|
+
"deploy could otherwise silently reattach to this "
|
|
1190
|
+
"volume's old data. Resolve the error above and "
|
|
1191
|
+
"re-run `lablink destroy`."
|
|
1192
|
+
)
|
|
1193
|
+
raise SystemExit(1)
|
|
1194
|
+
shutil.rmtree(target)
|
|
1195
|
+
console.print(f"[green]Removed {target}.[/green]")
|
|
1196
|
+
else:
|
|
1197
|
+
console.print(f"[green]Stack torn down (data preserved in {target}).[/green]")
|
|
1198
|
+
|
|
1199
|
+
console.print(
|
|
1200
|
+
"\n[bold]Reminder:[/bold] each BYO client box still has "
|
|
1201
|
+
"`lablink-client` running.\n"
|
|
1202
|
+
"Run [bold]lablink client unregister[/bold] on each box to clean up."
|
|
1203
|
+
)
|